diff --git a/.dockerignore b/.dockerignore index fcc890f76..40ff6c49f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,11 @@ conf/stack screenshots +tools test/data/registry venv .git .gitignore Bobfile README.md -license.py requirements-nover.txt -run-local.sh +run-local.sh \ No newline at end of file diff --git a/Dockerfile.buildworker b/Dockerfile.buildworker index c18c24589..159c7867c 100644 --- a/Dockerfile.buildworker +++ b/Dockerfile.buildworker @@ -1,13 +1,13 @@ -FROM phusion/baseimage:0.9.11 +FROM phusion/baseimage:0.9.13 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 06AUG2014 +RUN apt-get update # 10SEP2014 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev libpq-dev # Build the python dependencies ADD requirements.txt requirements.txt diff --git a/Dockerfile.web b/Dockerfile.web index c9ba36823..605b088b3 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,13 +1,13 @@ -FROM phusion/baseimage:0.9.11 +FROM phusion/baseimage:0.9.13 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 06AUG2014 +RUN apt-get update # 10SEP2014 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap2-dev libsasl2-dev libpq-dev # Build the python dependencies ADD requirements.txt requirements.txt @@ -30,6 +30,7 @@ RUN cd grunt && npm install RUN cd grunt && grunt ADD conf/init/svlogd_config /svlogd_config +ADD conf/init/doupdatelimits.sh /etc/my_init.d/ ADD conf/init/preplogsdir.sh /etc/my_init.d/ ADD conf/init/runmigration.sh /etc/my_init.d/ @@ -37,9 +38,7 @@ ADD conf/init/gunicorn /etc/service/gunicorn ADD conf/init/nginx /etc/service/nginx ADD conf/init/diffsworker /etc/service/diffsworker ADD conf/init/notificationworker /etc/service/notificationworker - -# TODO: Remove this after the prod CL push -ADD conf/init/webhookworker /etc/service/webhookworker +ADD conf/init/buildlogsarchiver /etc/service/buildlogsarchiver # Download any external libs. RUN mkdir static/fonts static/ldn @@ -48,7 +47,7 @@ RUN venv/bin/python -m external_libraries # Run the tests RUN TEST=true venv/bin/python -m unittest discover -VOLUME ["/conf/stack", "/var/log", "/datastorage"] +VOLUME ["/conf/stack", "/var/log", "/datastorage", "/tmp"] EXPOSE 443 80 diff --git a/app.py b/app.py index 78746fbcf..8f0a57d62 100644 --- a/app.py +++ b/app.py @@ -1,8 +1,9 @@ import logging import os import json +import yaml -from flask import Flask +from flask import Flask as BaseFlask, Config as BaseConfig from flask.ext.principal import Principal from flask.ext.login import LoginManager from flask.ext.mail import Mail @@ -19,13 +20,40 @@ from util.exceptionlog import Sentry from util.queuemetrics import QueueMetrics from data.billing import Billing from data.buildlogs import BuildLogs +from data.archivedlogs import LogArchive from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule -from license import load_license from datetime import datetime -OVERRIDE_CONFIG_FILENAME = 'conf/stack/config.py' +class Config(BaseConfig): + """ Flask config enhanced with a `from_yamlfile` method """ + + def from_yamlfile(self, config_file): + with open(config_file) as f: + c = yaml.load(f) + if not c: + logger.debug('Empty YAML config file') + return + + if isinstance(c, str): + raise Exception('Invalid YAML config file: ' + str(c)) + + for key in c.iterkeys(): + if key.isupper(): + self[key] = c[key] + +class Flask(BaseFlask): + """ Extends the Flask class to implement our custom Config class. """ + + def make_config(self, instance_relative=False): + root_path = self.instance_path if instance_relative else self.root_path + return Config(root_path, self.default_config) + + +OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' +OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py' + OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' LICENSE_FILENAME = 'conf/stack/license.enc' @@ -43,22 +71,17 @@ else: logger.debug('Loading default config.') app.config.from_object(DefaultConfig()) - if os.path.exists(OVERRIDE_CONFIG_FILENAME): - logger.debug('Applying config file: %s', OVERRIDE_CONFIG_FILENAME) - app.config.from_pyfile(OVERRIDE_CONFIG_FILENAME) + if os.path.exists(OVERRIDE_CONFIG_PY_FILENAME): + logger.debug('Applying config file: %s', OVERRIDE_CONFIG_PY_FILENAME) + app.config.from_pyfile(OVERRIDE_CONFIG_PY_FILENAME) + + if os.path.exists(OVERRIDE_CONFIG_YAML_FILENAME): + logger.debug('Applying config file: %s', OVERRIDE_CONFIG_YAML_FILENAME) + app.config.from_yamlfile(OVERRIDE_CONFIG_YAML_FILENAME) environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}')) app.config.update(environ_config) - logger.debug('Applying license config from: %s', LICENSE_FILENAME) - try: - app.config.update(load_license(LICENSE_FILENAME)) - except IOError: - raise RuntimeError('License file %s not found; please check your configuration' % LICENSE_FILENAME) - - if app.config.get('LICENSE_EXPIRATION', datetime.min) < datetime.utcnow(): - raise RuntimeError('License has expired, please contact support@quay.io') - features.import_features(app.config) Principal(app, use_sessions=False) @@ -66,7 +89,8 @@ Principal(app, use_sessions=False) login_manager = LoginManager(app) mail = Mail(app) storage = Storage(app) -userfiles = Userfiles(app) +userfiles = Userfiles(app, storage) +log_archive = LogArchive(app, storage) analytics = Analytics(app) billing = Billing(app) sentry = Sentry(app) diff --git a/application.py b/application.py index 2fb79835b..493b34fed 100644 --- a/application.py +++ b/application.py @@ -7,7 +7,7 @@ from peewee import Proxy from app import app as application from flask import request, Request from util.names import urn_generator -from data.model import db as model_db, read_slave +from data.database import db as model_db, read_slave # Turn off debug logging for boto logging.getLogger('boto').setLevel(logging.CRITICAL) diff --git a/auth/auth.py b/auth/auth.py index 3616792ad..ed0c8d82a 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -25,7 +25,7 @@ def _load_user_from_cookie(): if not current_user.is_anonymous(): logger.debug('Loading user from cookie: %s', current_user.get_id()) set_authenticated_user_deferred(current_user.get_id()) - loaded = QuayDeferredPermissionUser(current_user.get_id(), 'username', {scopes.DIRECT_LOGIN}) + loaded = QuayDeferredPermissionUser(current_user.get_id(), 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=loaded) return current_user.db_user() return None @@ -58,12 +58,10 @@ def _validate_and_apply_oauth_token(token): set_authenticated_user(validated.authorized_user) set_validated_oauth_token(validated) - new_identity = QuayDeferredPermissionUser(validated.authorized_user.username, 'username', - scope_set) + new_identity = QuayDeferredPermissionUser(validated.authorized_user.id, 'user_db_id', scope_set) identity_changed.send(app, identity=new_identity) - def process_basic_auth(auth): normalized = [part.strip() for part in auth.split(' ') if part] if normalized[0].lower() != 'basic' or len(normalized) != 2: @@ -100,8 +98,7 @@ def process_basic_auth(auth): logger.debug('Successfully validated robot: %s' % credentials[0]) set_authenticated_user(robot) - deferred_robot = QuayDeferredPermissionUser(robot.username, 'username', - {scopes.DIRECT_LOGIN}) + deferred_robot = QuayDeferredPermissionUser(robot.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=deferred_robot) return except model.InvalidRobotException: @@ -114,7 +111,7 @@ def process_basic_auth(auth): logger.debug('Successfully validated user: %s' % authenticated.username) set_authenticated_user(authenticated) - new_identity = QuayDeferredPermissionUser(authenticated.username, 'username', + new_identity = QuayDeferredPermissionUser(authenticated.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) return @@ -135,8 +132,15 @@ def process_token(auth): logger.warning('Invalid token format: %s' % auth) abort(401, message='Invalid token format: %(auth)s', issue='invalid-auth-token', auth=auth) - token_vals = {val[0]: val[1] for val in + def safe_get(lst, index, default_value): + try: + return lst[index] + except IndexError: + return default_value + + token_vals = {val[0]: safe_get(val, 1, '') for val in (detail.split('=') for detail in token_details)} + if 'signature' not in token_vals: logger.warning('Token does not contain signature: %s' % auth) abort(401, message='Token does not contain a valid signature: %(auth)s', diff --git a/auth/auth_context.py b/auth/auth_context.py index b97ffa02d..6c587f901 100644 --- a/auth/auth_context.py +++ b/auth/auth_context.py @@ -10,13 +10,13 @@ logger = logging.getLogger(__name__) def get_authenticated_user(): user = getattr(_request_ctx_stack.top, 'authenticated_user', None) if not user: - username = getattr(_request_ctx_stack.top, 'authenticated_username', None) - if not username: - logger.debug('No authenticated user or deferred username.') + db_id = getattr(_request_ctx_stack.top, 'authenticated_db_id', None) + if not db_id: + logger.debug('No authenticated user or deferred database id.') return None logger.debug('Loading deferred authenticated user.') - loaded = model.get_user(username) + loaded = model.get_user_by_id(db_id) set_authenticated_user(loaded) user = loaded @@ -30,10 +30,10 @@ def set_authenticated_user(user_or_robot): ctx.authenticated_user = user_or_robot -def set_authenticated_user_deferred(username_or_robotname): - logger.debug('Deferring loading of authenticated user object: %s', username_or_robotname) +def set_authenticated_user_deferred(user_or_robot_db_id): + logger.debug('Deferring loading of authenticated user object: %s', user_or_robot_db_id) ctx = _request_ctx_stack.top - ctx.authenticated_username = username_or_robotname + ctx.authenticated_db_id = user_or_robot_db_id def get_validated_oauth_token(): diff --git a/auth/permissions.py b/auth/permissions.py index 2b27f9583..eb9059c22 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -58,8 +58,8 @@ SCOPE_MAX_USER_ROLES.update({ class QuayDeferredPermissionUser(Identity): - def __init__(self, id, auth_type, scopes): - super(QuayDeferredPermissionUser, self).__init__(id, auth_type) + def __init__(self, db_id, auth_type, scopes): + super(QuayDeferredPermissionUser, self).__init__(db_id, auth_type) self._permissions_loaded = False self._scope_set = scopes @@ -88,7 +88,7 @@ class QuayDeferredPermissionUser(Identity): def can(self, permission): if not self._permissions_loaded: logger.debug('Loading user permissions after deferring.') - user_object = model.get_user(self.id) + user_object = model.get_user_by_id(self.id) # Add the superuser need, if applicable. if (user_object.username is not None and @@ -112,7 +112,7 @@ class QuayDeferredPermissionUser(Identity): # Add repository permissions for perm in model.get_all_user_permissions(user_object): - repo_grant = _RepositoryNeed(perm.repository.namespace, perm.repository.name, + repo_grant = _RepositoryNeed(perm.repository.namespace_user.username, perm.repository.name, self._repo_role_for_scopes(perm.role.name)) logger.debug('User added permission: {0}'.format(repo_grant)) self.provides.add(repo_grant) @@ -230,16 +230,16 @@ def on_identity_loaded(sender, identity): if isinstance(identity, QuayDeferredPermissionUser): logger.debug('Deferring permissions for user: %s', identity.id) - elif identity.auth_type == 'username': + elif identity.auth_type == 'user_db_id': logger.debug('Switching username permission to deferred object: %s', identity.id) - switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'username', {scopes.DIRECT_LOGIN}) + switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=switch_to_deferred) elif identity.auth_type == 'token': logger.debug('Loading permissions for token: %s', identity.id) token_data = model.load_token_data(identity.id) - repo_grant = _RepositoryNeed(token_data.repository.namespace, + repo_grant = _RepositoryNeed(token_data.repository.namespace_user.username, token_data.repository.name, token_data.role.name) logger.debug('Delegate token added permission: {0}'.format(repo_grant)) diff --git a/conf/gunicorn_config.py b/conf/gunicorn_config.py index 4d9d50499..ca8ad5363 100644 --- a/conf/gunicorn_config.py +++ b/conf/gunicorn_config.py @@ -1,5 +1,5 @@ bind = 'unix:/tmp/gunicorn.sock' -workers = 8 +workers = 16 worker_class = 'gevent' timeout = 2000 logconfig = 'conf/logging.conf' diff --git a/conf/init/buildlogsarchiver/log/run b/conf/init/buildlogsarchiver/log/run new file mode 100755 index 000000000..c35fb1fb9 --- /dev/null +++ b/conf/init/buildlogsarchiver/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/buildlogsarchiver/ \ No newline at end of file diff --git a/conf/init/buildlogsarchiver/run b/conf/init/buildlogsarchiver/run new file mode 100755 index 000000000..df3d4b05f --- /dev/null +++ b/conf/init/buildlogsarchiver/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting build logs archiver worker' + +cd / +venv/bin/python -m workers.buildlogsarchiver 2>&1 + +echo 'Diffs worker exited' \ No newline at end of file diff --git a/conf/init/doupdatelimits.sh b/conf/init/doupdatelimits.sh new file mode 100755 index 000000000..603559de0 --- /dev/null +++ b/conf/init/doupdatelimits.sh @@ -0,0 +1,5 @@ +#! /bin/bash +set -e + +# Update the connection limit +sysctl -w net.core.somaxconn=1024 \ No newline at end of file diff --git a/conf/init/webhookworker/log/run b/conf/init/webhookworker/log/run deleted file mode 100755 index 6738f16f8..000000000 --- a/conf/init/webhookworker/log/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec svlogd -t /var/log/webhookworker/ \ No newline at end of file diff --git a/conf/init/webhookworker/run b/conf/init/webhookworker/run deleted file mode 100755 index 04521552a..000000000 --- a/conf/init/webhookworker/run +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/bash - -echo 'Starting webhook worker' - -cd / -venv/bin/python -m workers.webhookworker - -echo 'Webhook worker exited' \ No newline at end of file diff --git a/conf/server-base.conf b/conf/server-base.conf index 6aeaa689e..a13cf1424 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -1,4 +1,4 @@ -client_max_body_size 8G; +client_max_body_size 20G; client_body_temp_path /var/log/nginx/client_body 1 2; server_name _; diff --git a/config.py b/config.py index a903fa29a..6742d1a43 100644 --- a/config.py +++ b/config.py @@ -19,7 +19,7 @@ def build_requests_session(): CLIENT_WHITELIST = ['SERVER_HOSTNAME', 'PREFERRED_URL_SCHEME', 'GITHUB_CLIENT_ID', 'GITHUB_LOGIN_CLIENT_ID', 'MIXPANEL_KEY', 'STRIPE_PUBLISHABLE_KEY', 'ENTERPRISE_LOGO_URL', 'SENTRY_PUBLIC_DSN', 'AUTHENTICATION_TYPE', - 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT'] + 'REGISTRY_TITLE', 'REGISTRY_TITLE_SHORT', 'GOOGLE_LOGIN_CLIENT_ID'] def getFrontendVisibleConfig(config_dict): @@ -80,19 +80,15 @@ class DefaultConfig(object): AUTHENTICATION_TYPE = 'Database' # Build logs - BUILDLOGS_REDIS_HOSTNAME = 'logs.quay.io' + BUILDLOGS_REDIS = {'host': 'logs.quay.io'} BUILDLOGS_OPTIONS = [] # Real-time user events - USER_EVENTS_REDIS_HOSTNAME = 'logs.quay.io' + USER_EVENTS_REDIS = {'host': 'logs.quay.io'} # Stripe config BILLING_TYPE = 'FakeStripe' - # Userfiles - USERFILES_TYPE = 'LocalUserfiles' - USERFILES_PATH = 'test/data/registry/userfiles' - # Analytics ANALYTICS_TYPE = 'FakeAnalytics' @@ -115,6 +111,13 @@ class DefaultConfig(object): GITHUB_LOGIN_CLIENT_ID = '' GITHUB_LOGIN_CLIENT_SECRET = '' + # Google Config. + GOOGLE_TOKEN_URL = 'https://accounts.google.com/o/oauth2/token' + GOOGLE_USER_URL = 'https://www.googleapis.com/oauth2/v1/userinfo' + + GOOGLE_LOGIN_CLIENT_ID = '' + GOOGLE_LOGIN_CLIENT_SECRET = '' + # Requests based HTTP client with a large request pool HTTPCLIENT = build_requests_session() @@ -144,6 +147,9 @@ class DefaultConfig(object): # Feature Flag: Whether GitHub login is supported. FEATURE_GITHUB_LOGIN = False + # Feature Flag: Whether Google login is supported. + FEATURE_GOOGLE_LOGIN = False + # Feature flag, whether to enable olark chat FEATURE_OLARK_CHAT = False @@ -153,9 +159,26 @@ class DefaultConfig(object): # Feature Flag: Whether to support GitHub build triggers. FEATURE_GITHUB_BUILD = False + # Feature Flag: Dockerfile build support. + FEATURE_BUILD_SUPPORT = True + + # Feature Flag: Whether emails are enabled. + FEATURE_MAILING = True + + # Feature Flag: Whether users can be created (by non-super users). + FEATURE_USER_CREATION = True + DISTRIBUTED_STORAGE_CONFIG = { 'local_eu': ['LocalStorage', {'storage_path': 'test/data/registry/eu'}], 'local_us': ['LocalStorage', {'storage_path': 'test/data/registry/us'}], } DISTRIBUTED_STORAGE_PREFERENCE = ['local_us'] + + # Userfiles + USERFILES_LOCATION = 'local_us' + USERFILES_PATH = 'userfiles/' + + # Build logs archive + LOG_ARCHIVE_LOCATION = 'local_us' + LOG_ARCHIVE_PATH = 'logarchive/' diff --git a/data/archivedlogs.py b/data/archivedlogs.py new file mode 100644 index 000000000..e190b9782 --- /dev/null +++ b/data/archivedlogs.py @@ -0,0 +1,56 @@ +import logging + +from gzip import GzipFile +from flask import send_file, abort +from cStringIO import StringIO + +from data.userfiles import DelegateUserfiles, UserfilesHandlers + + +JSON_MIMETYPE = 'application/json' + + +logger = logging.getLogger(__name__) + + +class LogArchiveHandlers(UserfilesHandlers): + def get(self, file_id): + path = self._files.get_file_id_path(file_id) + try: + with self._storage.stream_read_file(self._locations, path) as gzip_stream: + with GzipFile(fileobj=gzip_stream) as unzipped: + unzipped_buffer = StringIO(unzipped.read()) + return send_file(unzipped_buffer, mimetype=JSON_MIMETYPE) + except IOError: + abort(404) + + +class LogArchive(object): + def __init__(self, app=None, distributed_storage=None): + self.app = app + if app is not None: + self.state = self.init_app(app, distributed_storage) + else: + self.state = None + + def init_app(self, app, distributed_storage): + location = app.config.get('LOG_ARCHIVE_LOCATION') + path = app.config.get('LOG_ARCHIVE_PATH', None) + + handler_name = 'logarchive_handlers' + + log_archive = DelegateUserfiles(app, distributed_storage, location, path, handler_name) + + app.add_url_rule('/logarchive/', + view_func=LogArchiveHandlers.as_view(handler_name, + distributed_storage=distributed_storage, + location=location, + files=log_archive)) + + # register extension with app + app.extensions = getattr(app, 'extensions', {}) + app.extensions['log_archive'] = log_archive + return log_archive + + def __getattr__(self, name): + return getattr(self.state, name, None) diff --git a/data/billing.py b/data/billing.py index 4847dd3f8..e1510c054 100644 --- a/data/billing.py +++ b/data/billing.py @@ -3,6 +3,8 @@ import stripe from datetime import datetime, timedelta from calendar import timegm +from util.morecollections import AttrDict + PLANS = [ # Deprecated Plans { @@ -118,20 +120,6 @@ def get_plan(plan_id): return None -class AttrDict(dict): - def __init__(self, *args, **kwargs): - super(AttrDict, self).__init__(*args, **kwargs) - self.__dict__ = self - - @classmethod - def deep_copy(cls, attr_dict): - copy = AttrDict(attr_dict) - for key, value in copy.items(): - if isinstance(value, AttrDict): - copy[key] = cls.deep_copy(value) - return copy - - class FakeStripe(object): class Customer(AttrDict): FAKE_PLAN = AttrDict({ diff --git a/data/buildlogs.py b/data/buildlogs.py index 2ccd03899..6e24f501b 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -2,6 +2,11 @@ import redis import json from util.dynamic import import_class +from datetime import timedelta + + +ONE_DAY = timedelta(days=1) + class BuildStatusRetrievalError(Exception): pass @@ -11,8 +16,8 @@ class RedisBuildLogs(object): COMMAND = 'command' PHASE = 'phase' - def __init__(self, redis_host): - self._redis = redis.StrictRedis(host=redis_host) + def __init__(self, redis_config): + self._redis = redis.StrictRedis(socket_connect_timeout=5, **redis_config) @staticmethod def _logs_key(build_id): @@ -25,7 +30,7 @@ class RedisBuildLogs(object): """ return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - def append_log_message(self, build_id, log_message, log_type=None): + def append_log_message(self, build_id, log_message, log_type=None, log_data=None): """ Wraps the message in an envelope and push it to the end of the log entry list and returns the index at which it was inserted. @@ -37,6 +42,9 @@ class RedisBuildLogs(object): if log_type: log_obj['type'] = log_type + if log_data: + log_obj['data'] = log_data + return self._redis.rpush(self._logs_key(build_id), json.dumps(log_obj)) - 1 def get_log_entries(self, build_id, start_index): @@ -51,6 +59,13 @@ class RedisBuildLogs(object): except redis.ConnectionError: raise BuildStatusRetrievalError('Cannot retrieve build logs') + def expire_log_entries(self, build_id): + """ + Sets the log entry to expire in 1 day. + """ + self._redis.expire(self._logs_key(build_id), ONE_DAY) + + @staticmethod def _status_key(build_id): return 'builds/%s/status' % build_id @@ -89,7 +104,13 @@ class BuildLogs(object): self.state = None def init_app(self, app): - buildlogs_hostname = app.config.get('BUILDLOGS_REDIS_HOSTNAME') + buildlogs_config = app.config.get('BUILDLOGS_REDIS') + if not buildlogs_config: + # This is the old key name. + buildlogs_config = { + 'host': app.config.get('BUILDLOGS_REDIS_HOSTNAME') + } + buildlogs_options = app.config.get('BUILDLOGS_OPTIONS', []) buildlogs_import = app.config.get('BUILDLOGS_MODULE_AND_CLASS', None) @@ -98,7 +119,7 @@ class BuildLogs(object): else: klass = import_class(buildlogs_import[0], buildlogs_import[1]) - buildlogs = klass(buildlogs_hostname, *buildlogs_options) + buildlogs = klass(buildlogs_config, *buildlogs_options) # register extension with app app.extensions = getattr(app, 'extensions', {}) @@ -106,4 +127,4 @@ class BuildLogs(object): return buildlogs def __getattr__(self, name): - return getattr(self.state, name, None) \ No newline at end of file + return getattr(self.state, name, None) diff --git a/data/database.py b/data/database.py index 76a0af9df..aba62b88b 100644 --- a/data/database.py +++ b/data/database.py @@ -8,7 +8,7 @@ from peewee import * from data.read_slave import ReadSlaveModel from sqlalchemy.engine.url import make_url from urlparse import urlparse - +from util.names import urn_generator logger = logging.getLogger(__name__) @@ -17,10 +17,28 @@ SCHEME_DRIVERS = { 'mysql': MySQLDatabase, 'mysql+pymysql': MySQLDatabase, 'sqlite': SqliteDatabase, + 'postgresql': PostgresqlDatabase, + 'postgresql+psycopg2': PostgresqlDatabase, } +SCHEME_RANDOM_FUNCTION = { + 'mysql': fn.Rand, + 'mysql+pymysql': fn.Rand, + 'sqlite': fn.Random, + 'postgresql': fn.Random, + 'postgresql+psycopg2': fn.Random, +} + +class CallableProxy(Proxy): + def __call__(self, *args, **kwargs): + if self.obj is None: + raise AttributeError('Cannot use uninitialized Proxy.') + return self.obj(*args, **kwargs) + db = Proxy() read_slave = Proxy() +db_random_func = CallableProxy() + def _db_from_url(url, db_kwargs): parsed_url = make_url(url) @@ -32,15 +50,19 @@ def _db_from_url(url, db_kwargs): if parsed_url.username: db_kwargs['user'] = parsed_url.username if parsed_url.password: - db_kwargs['passwd'] = parsed_url.password + db_kwargs['password'] = parsed_url.password return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs) + def configure(config_object): db_kwargs = dict(config_object['DB_CONNECTION_ARGS']) write_db_uri = config_object['DB_URI'] db.initialize(_db_from_url(write_db_uri, db_kwargs)) + parsed_write_uri = make_url(write_db_uri) + db_random_func.initialize(SCHEME_RANDOM_FUNCTION[parsed_write_uri.drivername]) + read_slave_uri = config_object.get('DB_READ_SLAVE_URI', None) if read_slave_uri is not None: read_slave.initialize(_db_from_url(read_slave_uri, db_kwargs)) @@ -74,6 +96,8 @@ class User(BaseModel): organization = BooleanField(default=False, index=True) robot = BooleanField(default=False, index=True) invoice_email = BooleanField(default=False) + invalid_login_attempts = IntegerField(default=0) + last_invalid_login = DateTimeField(default=datetime.utcnow) class TeamRole(BaseModel): @@ -108,6 +132,15 @@ class TeamMember(BaseModel): ) +class TeamMemberInvite(BaseModel): + # Note: Either user OR email will be filled in, but not both. + user = ForeignKeyField(User, index=True, null=True) + email = CharField(null=True) + team = ForeignKeyField(Team, index=True) + inviter = ForeignKeyField(User, related_name='inviter') + invite_token = CharField(default=urn_generator(['teaminvite'])) + + class LoginService(BaseModel): name = CharField(unique=True, index=True) @@ -116,6 +149,7 @@ class FederatedLogin(BaseModel): user = ForeignKeyField(User, index=True) service = ForeignKeyField(LoginService, index=True) service_ident = CharField() + metadata_json = TextField(default='{}') class Meta: database = db @@ -134,7 +168,7 @@ class Visibility(BaseModel): class Repository(BaseModel): - namespace = CharField() + namespace_user = ForeignKeyField(User) name = CharField() visibility = ForeignKeyField(Visibility) description = TextField(null=True) @@ -145,7 +179,7 @@ class Repository(BaseModel): read_slaves = (read_slave,) indexes = ( # create a unique index on namespace and name - (('namespace', 'name'), True), + (('namespace_user', 'name'), True), ) @@ -229,6 +263,7 @@ class ImageStorage(BaseModel): comment = TextField(null=True) command = TextField(null=True) image_size = BigIntegerField(null=True) + uncompressed_size = BigIntegerField(null=True) uploading = BooleanField(default=True, null=True) @@ -284,6 +319,16 @@ class RepositoryTag(BaseModel): ) +class BUILD_PHASE(object): + """ Build phases enum """ + ERROR = 'error' + UNPACKING = 'unpacking' + PULLING = 'pulling' + BUILDING = 'building' + PUSHING = 'pushing' + COMPLETE = 'complete' + + class RepositoryBuild(BaseModel): uuid = CharField(default=uuid_generator, index=True) repository = ForeignKeyField(Repository, index=True) @@ -295,12 +340,13 @@ class RepositoryBuild(BaseModel): display_name = CharField() trigger = ForeignKeyField(RepositoryBuildTrigger, null=True, index=True) pull_robot = ForeignKeyField(User, null=True, related_name='buildpullrobot') + logs_archived = BooleanField(default=False) class QueueItem(BaseModel): queue_name = CharField(index=True, max_length=1024) body = TextField() - available_after = DateTimeField(default=datetime.now, index=True) + available_after = DateTimeField(default=datetime.utcnow, index=True) available = BooleanField(default=True, index=True) processing_expires = DateTimeField(null=True, index=True) retries_remaining = IntegerField(default=5) @@ -405,4 +451,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, OAuthApplication, OAuthAuthorizationCode, OAuthAccessToken, NotificationKind, Notification, ImageStorageLocation, ImageStoragePlacement, ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, - RepositoryAuthorizedEmail] + RepositoryAuthorizedEmail, TeamMemberInvite] diff --git a/data/migrations/env.py b/data/migrations/env.py index c267c2f50..d64cf4ee7 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -8,6 +8,7 @@ from peewee import SqliteDatabase from data.database import all_models, db from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata +from util.morecollections import AttrDict # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -23,6 +24,7 @@ fileConfig(config.config_file_name) # from myapp import mymodel # target_metadata = mymodel.Base.metadata target_metadata = gen_sqlalchemy_metadata(all_models) +tables = AttrDict(target_metadata.tables) # other values from the config, defined by the needs of env.py, # can be acquired: @@ -45,7 +47,7 @@ def run_migrations_offline(): context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True) with context.begin_transaction(): - context.run_migrations() + context.run_migrations(tables=tables) def run_migrations_online(): """Run migrations in 'online' mode. @@ -72,7 +74,7 @@ def run_migrations_online(): try: with context.begin_transaction(): - context.run_migrations() + context.run_migrations(tables=tables) finally: connection.close() diff --git a/data/migrations/script.py.mako b/data/migrations/script.py.mako index 95702017e..1b92f9f48 100644 --- a/data/migrations/script.py.mako +++ b/data/migrations/script.py.mako @@ -14,9 +14,9 @@ from alembic import op import sqlalchemy as sa ${imports if imports else ""} -def upgrade(): +def upgrade(tables): ${upgrades if upgrades else "pass"} -def downgrade(): +def downgrade(tables): ${downgrades if downgrades else "pass"} diff --git a/data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py b/data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py new file mode 100644 index 000000000..30ac75c96 --- /dev/null +++ b/data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py @@ -0,0 +1,24 @@ +"""Migrate registry namespaces to reference a user. + +Revision ID: 13da56878560 +Revises: 51d04d0e7e6f +Create Date: 2014-09-18 13:56:45.130455 + +""" + +# revision identifiers, used by Alembic. +revision = '13da56878560' +down_revision = '51d04d0e7e6f' + +from alembic import op +import sqlalchemy as sa + +from data.database import Repository, User + +def upgrade(tables): + # Add the namespace_user column, allowing it to be nullable + op.add_column('repository', sa.Column('namespace_user_id', sa.Integer(), sa.ForeignKey('user.id'))) + + +def downgrade(tables): + op.drop_column('repository', 'namespace_user_id') diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py new file mode 100644 index 000000000..2f6c60706 --- /dev/null +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -0,0 +1,35 @@ +"""add metadata field to external logins + +Revision ID: 1594a74a74ca +Revises: f42b0ea7a4d +Create Date: 2014-09-04 18:17:35.205698 + +""" + +# revision identifiers, used by Alembic. +revision = '1594a74a74ca' +down_revision = 'f42b0ea7a4d' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('federatedlogin', sa.Column('metadata_json', sa.Text(), nullable=False)) + ### end Alembic commands ### + + op.bulk_insert(tables.loginservice, + [ + {'id':4, 'name':'google'}, + ]) + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('federatedlogin', 'metadata_json') + ### end Alembic commands ### + + op.execute( + (tables.loginservice.delete() + .where(tables.loginservice.c.name == op.inline_literal('google'))) + ) diff --git a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py index ea36e3f57..d50c3a592 100644 --- a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py +++ b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(): +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_index('buildtriggerservice_name', table_name='buildtriggerservice') op.create_index('buildtriggerservice_name', 'buildtriggerservice', ['name'], unique=True) @@ -34,7 +34,7 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_index('visibility_name', table_name='visibility') op.create_index('visibility_name', 'visibility', ['name'], unique=False) diff --git a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py index 18c8bf654..e3be811b6 100644 --- a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py +++ b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py @@ -13,12 +13,8 @@ down_revision = '4b7ef0c7bdb2' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('externalnotificationmethod', sa.Column('id', sa.Integer(), nullable=False), @@ -26,7 +22,7 @@ def upgrade(): sa.PrimaryKeyConstraint('id') ) op.create_index('externalnotificationmethod_name', 'externalnotificationmethod', ['name'], unique=True) - op.bulk_insert(schema.tables['externalnotificationmethod'], + op.bulk_insert(tables.externalnotificationmethod, [ {'id':1, 'name':'quay_notification'}, {'id':2, 'name':'email'}, @@ -38,7 +34,7 @@ def upgrade(): sa.PrimaryKeyConstraint('id') ) op.create_index('externalnotificationevent_name', 'externalnotificationevent', ['name'], unique=True) - op.bulk_insert(schema.tables['externalnotificationevent'], + op.bulk_insert(tables.externalnotificationevent, [ {'id':1, 'name':'repo_push'}, {'id':2, 'name':'build_queued'}, @@ -77,7 +73,7 @@ def upgrade(): op.add_column(u'notification', sa.Column('dismissed', sa.Boolean(), nullable=False)) # Manually add the new notificationkind types - op.bulk_insert(schema.tables['notificationkind'], + op.bulk_insert(tables.notificationkind, [ {'id':5, 'name':'repo_push'}, {'id':6, 'name':'build_queued'}, @@ -87,7 +83,7 @@ def upgrade(): ]) # Manually add the new logentrykind types - op.bulk_insert(schema.tables['logentrykind'], + op.bulk_insert(tables.logentrykind, [ {'id':39, 'name':'add_repo_notification'}, {'id':40, 'name':'delete_repo_notification'}, @@ -97,61 +93,49 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column(u'notification', 'dismissed') - op.drop_index('repositorynotification_uuid', table_name='repositorynotification') - op.drop_index('repositorynotification_repository_id', table_name='repositorynotification') - op.drop_index('repositorynotification_method_id', table_name='repositorynotification') - op.drop_index('repositorynotification_event_id', table_name='repositorynotification') op.drop_table('repositorynotification') - op.drop_index('repositoryauthorizedemail_repository_id', table_name='repositoryauthorizedemail') - op.drop_index('repositoryauthorizedemail_email_repository_id', table_name='repositoryauthorizedemail') - op.drop_index('repositoryauthorizedemail_code', table_name='repositoryauthorizedemail') op.drop_table('repositoryauthorizedemail') - op.drop_index('externalnotificationevent_name', table_name='externalnotificationevent') op.drop_table('externalnotificationevent') - op.drop_index('externalnotificationmethod_name', table_name='externalnotificationmethod') op.drop_table('externalnotificationmethod') # Manually remove the notificationkind and logentrykind types - notificationkind = schema.tables['notificationkind'] op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('repo_push'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('repo_push'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_queued'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_queued'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_start'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_start'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_success'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_success'))) ) op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('build_failure'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('build_failure'))) ) op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('add_repo_notification'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('add_repo_notification'))) ) op.execute( - (logentrykind.delete() - .where(logentrykind.c.name == op.inline_literal('delete_repo_notification'))) + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('delete_repo_notification'))) ) ### end Alembic commands ### diff --git a/data/migrations/versions/34fd69f63809_add_support_for_build_log_migration.py b/data/migrations/versions/34fd69f63809_add_support_for_build_log_migration.py new file mode 100644 index 000000000..a731d0158 --- /dev/null +++ b/data/migrations/versions/34fd69f63809_add_support_for_build_log_migration.py @@ -0,0 +1,26 @@ +"""Add support for build log migration. + +Revision ID: 34fd69f63809 +Revises: 4a0c94399f38 +Create Date: 2014-09-12 11:50:09.217777 + +""" + +# revision identifiers, used by Alembic. +revision = '34fd69f63809' +down_revision = '4a0c94399f38' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorybuild', sa.Column('logs_archived', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('repositorybuild', 'logs_archived') + ### end Alembic commands ### diff --git a/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py new file mode 100644 index 000000000..6f40f4fc0 --- /dev/null +++ b/data/migrations/versions/3f4fe1194671_backfill_the_namespace_user_fields.py @@ -0,0 +1,26 @@ +"""Backfill the namespace_user fields. + +Revision ID: 3f4fe1194671 +Revises: 6f2ecf5afcf +Create Date: 2014-09-24 14:29:45.192179 + +""" + +# revision identifiers, used by Alembic. +revision = '3f4fe1194671' +down_revision = '6f2ecf5afcf' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + conn = op.get_bind() + conn.execute('update repository set namespace_user_id = (select id from user where user.username = repository.namespace) where namespace_user_id is NULL') + + op.create_index('repository_namespace_user_id_name', 'repository', ['namespace_user_id', 'name'], unique=True) + + +def downgrade(tables): + op.drop_constraint('fk_repository_namespace_user_id_user', table_name='repository', type_='foreignkey') + op.drop_index('repository_namespace_user_id_name', table_name='repository') diff --git a/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py new file mode 100644 index 000000000..f676bf972 --- /dev/null +++ b/data/migrations/versions/43e943c0639f_add_log_kind_for_regenerating_robot_.py @@ -0,0 +1,29 @@ +"""add log kind for regenerating robot tokens + +Revision ID: 43e943c0639f +Revises: 82297d834ad +Create Date: 2014-08-25 17:14:42.784518 + +""" + +# revision identifiers, used by Alembic. +revision = '43e943c0639f' +down_revision = '82297d834ad' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + op.bulk_insert(tables.logentrykind, + [ + {'id': 41, 'name':'regenerate_robot_token'}, + ]) + + +def downgrade(tables): + op.execute( + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('regenerate_robot_token'))) + + ) diff --git a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py index 6f516e9b9..eaa687c73 100644 --- a/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py +++ b/data/migrations/versions/47670cbeced_migrate_existing_webhooks_to_.py @@ -18,14 +18,14 @@ def get_id(query): conn = op.get_bind() return list(conn.execute(query, ()).fetchall())[0][0] -def upgrade(): +def upgrade(tables): conn = op.get_bind() - event_id = get_id('Select id From externalnotificationevent Where name="repo_push" Limit 1') - method_id = get_id('Select id From externalnotificationmethod Where name="webhook" Limit 1') + event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') + method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') conn.execute('Insert Into repositorynotification (uuid, repository_id, event_id, method_id, config_json) Select public_id, repository_id, %s, %s, parameters FROM webhook' % (event_id, method_id)) -def downgrade(): +def downgrade(tables): conn = op.get_bind() - event_id = get_id('Select id From externalnotificationevent Where name="repo_push" Limit 1') - method_id = get_id('Select id From externalnotificationmethod Where name="webhook" Limit 1') + event_id = get_id('Select id From externalnotificationevent Where name=\'repo_push\' Limit 1') + method_id = get_id('Select id From externalnotificationmethod Where name=\'webhook\' Limit 1') conn.execute('Insert Into webhook (public_id, repository_id, parameters) Select uuid, repository_id, config_json FROM repositorynotification Where event_id=%s And method_id=%s' % (event_id, method_id)) diff --git a/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py b/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py new file mode 100644 index 000000000..6b4160b19 --- /dev/null +++ b/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py @@ -0,0 +1,39 @@ +"""add new notification kinds + +Revision ID: 4a0c94399f38 +Revises: 1594a74a74ca +Create Date: 2014-08-28 16:17:01.898269 + +""" + +# revision identifiers, used by Alembic. +revision = '4a0c94399f38' +down_revision = '1594a74a74ca' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + op.bulk_insert(tables.externalnotificationmethod, + [ + {'id':4, 'name':'flowdock'}, + {'id':5, 'name':'hipchat'}, + {'id':6, 'name':'slack'}, + ]) + +def downgrade(tables): + op.execute( + (tables.externalnotificationmethod.delete() + .where(tables.externalnotificationmethod.c.name == op.inline_literal('flowdock'))) + ) + + op.execute( + (tables.externalnotificationmethod.delete() + .where(tables.externalnotificationmethod.c.name == op.inline_literal('hipchat'))) + ) + + op.execute( + (tables.externalnotificationmethod.delete() + .where(tables.externalnotificationmethod.c.name == op.inline_literal('slack'))) + ) diff --git a/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py b/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py index 9e5fff425..9f48ca6c6 100644 --- a/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py +++ b/data/migrations/versions/4b7ef0c7bdb2_add_the_maintenance_notification_type.py @@ -11,23 +11,18 @@ revision = '4b7ef0c7bdb2' down_revision = 'bcdde200a1b' from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - op.bulk_insert(schema.tables['notificationkind'], +def upgrade(tables): + op.bulk_insert(tables.notificationkind, [ {'id':4, 'name':'maintenance'}, ]) -def downgrade(): - notificationkind = schema.tables['notificationkind'] +def downgrade(tables): op.execute( - (notificationkind.delete() - .where(notificationkind.c.name == op.inline_literal('maintenance'))) + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('maintenance'))) ) diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py new file mode 100644 index 000000000..1ce802eca --- /dev/null +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -0,0 +1,28 @@ +"""Add brute force prevention metadata to the user table. + +Revision ID: 4fdb65816b8d +Revises: 43e943c0639f +Create Date: 2014-09-03 12:35:33.722435 + +""" + +# revision identifiers, used by Alembic. +revision = '4fdb65816b8d' +down_revision = '43e943c0639f' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('user', sa.Column('invalid_login_attempts', sa.Integer(), nullable=False, server_default="0")) + op.add_column('user', sa.Column('last_invalid_login', sa.DateTime(), nullable=False, server_default=sa.func.now())) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('user', 'last_invalid_login') + op.drop_column('user', 'invalid_login_attempts') + ### end Alembic commands ### diff --git a/data/migrations/versions/51d04d0e7e6f_email_invites_for_joining_a_team.py b/data/migrations/versions/51d04d0e7e6f_email_invites_for_joining_a_team.py new file mode 100644 index 000000000..c18335adb --- /dev/null +++ b/data/migrations/versions/51d04d0e7e6f_email_invites_for_joining_a_team.py @@ -0,0 +1,78 @@ +"""Email invites for joining a team. + +Revision ID: 51d04d0e7e6f +Revises: 34fd69f63809 +Create Date: 2014-09-15 23:51:35.478232 + +""" + +# revision identifiers, used by Alembic. +revision = '51d04d0e7e6f' +down_revision = '34fd69f63809' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('teammemberinvite', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('email', sa.String(length=255), nullable=True), + sa.Column('team_id', sa.Integer(), nullable=False), + sa.Column('inviter_id', sa.Integer(), nullable=False), + sa.Column('invite_token', sa.String(length=255), nullable=False), + sa.ForeignKeyConstraint(['inviter_id'], ['user.id'], ), + sa.ForeignKeyConstraint(['team_id'], ['team.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('teammemberinvite_inviter_id', 'teammemberinvite', ['inviter_id'], unique=False) + op.create_index('teammemberinvite_team_id', 'teammemberinvite', ['team_id'], unique=False) + op.create_index('teammemberinvite_user_id', 'teammemberinvite', ['user_id'], unique=False) + ### end Alembic commands ### + + # Manually add the new logentrykind types + op.bulk_insert(tables.logentrykind, + [ + {'id':42, 'name':'org_invite_team_member'}, + {'id':43, 'name':'org_team_member_invite_accepted'}, + {'id':44, 'name':'org_team_member_invite_declined'}, + {'id':45, 'name':'org_delete_team_member_invite'}, + ]) + + op.bulk_insert(tables.notificationkind, + [ + {'id':10, 'name':'org_team_invite'}, + ]) + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.execute( + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('org_invite_team_member'))) + ) + + op.execute( + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('org_team_member_invite_accepted'))) + ) + + op.execute( + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('org_team_member_invite_declined'))) + ) + + op.execute( + (tables.logentrykind.delete() + .where(tables.logentrykind.c.name == op.inline_literal('org_delete_team_member_invite'))) + ) + + op.execute( + (tables.notificationkind.delete() + .where(tables.notificationkind.c.name == op.inline_literal('org_team_invite'))) + ) + + op.drop_table('teammemberinvite') + ### end Alembic commands ### diff --git a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py index 23aaf506a..f67224645 100644 --- a/data/migrations/versions/5a07499ce53f_set_up_initial_database.py +++ b/data/migrations/versions/5a07499ce53f_set_up_initial_database.py @@ -11,14 +11,9 @@ revision = '5a07499ce53f' down_revision = None from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('loginservice', sa.Column('id', sa.Integer(), nullable=False), @@ -27,7 +22,7 @@ def upgrade(): ) op.create_index('loginservice_name', 'loginservice', ['name'], unique=True) - op.bulk_insert(schema.tables['loginservice'], + op.bulk_insert(tables.loginservice, [ {'id':1, 'name':'github'}, {'id':2, 'name':'quayrobot'}, @@ -66,7 +61,7 @@ def upgrade(): ) op.create_index('role_name', 'role', ['name'], unique=False) - op.bulk_insert(schema.tables['role'], + op.bulk_insert(tables.role, [ {'id':1, 'name':'admin'}, {'id':2, 'name':'write'}, @@ -80,7 +75,7 @@ def upgrade(): ) op.create_index('logentrykind_name', 'logentrykind', ['name'], unique=False) - op.bulk_insert(schema.tables['logentrykind'], + op.bulk_insert(tables.logentrykind, [ {'id':1, 'name':'account_change_plan'}, {'id':2, 'name':'account_change_cc'}, @@ -136,7 +131,7 @@ def upgrade(): ) op.create_index('notificationkind_name', 'notificationkind', ['name'], unique=False) - op.bulk_insert(schema.tables['notificationkind'], + op.bulk_insert(tables.notificationkind, [ {'id':1, 'name':'password_required'}, {'id':2, 'name':'over_private_usage'}, @@ -150,7 +145,7 @@ def upgrade(): ) op.create_index('teamrole_name', 'teamrole', ['name'], unique=False) - op.bulk_insert(schema.tables['teamrole'], + op.bulk_insert(tables.teamrole, [ {'id':1, 'name':'admin'}, {'id':2, 'name':'creator'}, @@ -164,7 +159,7 @@ def upgrade(): ) op.create_index('visibility_name', 'visibility', ['name'], unique=False) - op.bulk_insert(schema.tables['visibility'], + op.bulk_insert(tables.visibility, [ {'id':1, 'name':'public'}, {'id':2, 'name':'private'}, @@ -194,7 +189,7 @@ def upgrade(): ) op.create_index('buildtriggerservice_name', 'buildtriggerservice', ['name'], unique=False) - op.bulk_insert(schema.tables['buildtriggerservice'], + op.bulk_insert(tables.buildtriggerservice, [ {'id':1, 'name':'github'}, ]) @@ -203,7 +198,7 @@ def upgrade(): sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_id', sa.Integer(), nullable=False), sa.Column('service_id', sa.Integer(), nullable=False), - sa.Column('service_ident', sa.String(length=255, collation='utf8_general_ci'), nullable=False), + sa.Column('service_ident', sa.String(length=255), nullable=False), sa.ForeignKeyConstraint(['service_id'], ['loginservice.id'], ), sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), sa.PrimaryKeyConstraint('id') @@ -375,7 +370,7 @@ def upgrade(): sa.Column('command', sa.Text(), nullable=True), sa.Column('repository_id', sa.Integer(), nullable=False), sa.Column('image_size', sa.BigInteger(), nullable=True), - sa.Column('ancestors', sa.String(length=60535, collation='latin1_swedish_ci'), nullable=True), + sa.Column('ancestors', sa.String(length=60535), nullable=True), sa.Column('storage_id', sa.Integer(), nullable=True), sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], ), sa.ForeignKeyConstraint(['storage_id'], ['imagestorage.id'], ), @@ -490,119 +485,34 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.drop_index('repositorybuild_uuid', table_name='repositorybuild') - op.drop_index('repositorybuild_trigger_id', table_name='repositorybuild') - op.drop_index('repositorybuild_resource_key', table_name='repositorybuild') - op.drop_index('repositorybuild_repository_id', table_name='repositorybuild') - op.drop_index('repositorybuild_pull_robot_id', table_name='repositorybuild') - op.drop_index('repositorybuild_access_token_id', table_name='repositorybuild') op.drop_table('repositorybuild') - op.drop_index('repositorybuildtrigger_write_token_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_service_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_repository_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_pull_robot_id', table_name='repositorybuildtrigger') - op.drop_index('repositorybuildtrigger_connected_user_id', table_name='repositorybuildtrigger') op.drop_table('repositorybuildtrigger') - op.drop_index('logentry_repository_id', table_name='logentry') - op.drop_index('logentry_performer_id', table_name='logentry') - op.drop_index('logentry_kind_id', table_name='logentry') - op.drop_index('logentry_datetime', table_name='logentry') - op.drop_index('logentry_account_id', table_name='logentry') - op.drop_index('logentry_access_token_id', table_name='logentry') op.drop_table('logentry') - op.drop_index('repositorytag_repository_id_name', table_name='repositorytag') - op.drop_index('repositorytag_repository_id', table_name='repositorytag') - op.drop_index('repositorytag_image_id', table_name='repositorytag') op.drop_table('repositorytag') - op.drop_index('permissionprototype_role_id', table_name='permissionprototype') - op.drop_index('permissionprototype_org_id_activating_user_id', table_name='permissionprototype') - op.drop_index('permissionprototype_org_id', table_name='permissionprototype') - op.drop_index('permissionprototype_delegate_user_id', table_name='permissionprototype') - op.drop_index('permissionprototype_delegate_team_id', table_name='permissionprototype') - op.drop_index('permissionprototype_activating_user_id', table_name='permissionprototype') op.drop_table('permissionprototype') - op.drop_index('image_storage_id', table_name='image') - op.drop_index('image_repository_id_docker_image_id', table_name='image') - op.drop_index('image_repository_id', table_name='image') - op.drop_index('image_ancestors', table_name='image') op.drop_table('image') - op.drop_index('oauthauthorizationcode_code', table_name='oauthauthorizationcode') - op.drop_index('oauthauthorizationcode_application_id', table_name='oauthauthorizationcode') op.drop_table('oauthauthorizationcode') - op.drop_index('webhook_repository_id', table_name='webhook') - op.drop_index('webhook_public_id', table_name='webhook') op.drop_table('webhook') - op.drop_index('teammember_user_id_team_id', table_name='teammember') - op.drop_index('teammember_user_id', table_name='teammember') - op.drop_index('teammember_team_id', table_name='teammember') op.drop_table('teammember') - op.drop_index('oauthaccesstoken_uuid', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_refresh_token', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_authorized_user_id', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_application_id', table_name='oauthaccesstoken') - op.drop_index('oauthaccesstoken_access_token', table_name='oauthaccesstoken') op.drop_table('oauthaccesstoken') - op.drop_index('repositorypermission_user_id_repository_id', table_name='repositorypermission') - op.drop_index('repositorypermission_user_id', table_name='repositorypermission') - op.drop_index('repositorypermission_team_id_repository_id', table_name='repositorypermission') - op.drop_index('repositorypermission_team_id', table_name='repositorypermission') - op.drop_index('repositorypermission_role_id', table_name='repositorypermission') - op.drop_index('repositorypermission_repository_id', table_name='repositorypermission') op.drop_table('repositorypermission') - op.drop_index('accesstoken_role_id', table_name='accesstoken') - op.drop_index('accesstoken_repository_id', table_name='accesstoken') - op.drop_index('accesstoken_code', table_name='accesstoken') op.drop_table('accesstoken') - op.drop_index('repository_visibility_id', table_name='repository') - op.drop_index('repository_namespace_name', table_name='repository') op.drop_table('repository') - op.drop_index('team_role_id', table_name='team') - op.drop_index('team_organization_id', table_name='team') - op.drop_index('team_name_organization_id', table_name='team') - op.drop_index('team_name', table_name='team') op.drop_table('team') - op.drop_index('emailconfirmation_user_id', table_name='emailconfirmation') - op.drop_index('emailconfirmation_code', table_name='emailconfirmation') op.drop_table('emailconfirmation') - op.drop_index('notification_uuid', table_name='notification') - op.drop_index('notification_target_id', table_name='notification') - op.drop_index('notification_kind_id', table_name='notification') - op.drop_index('notification_created', table_name='notification') op.drop_table('notification') - op.drop_index('oauthapplication_organization_id', table_name='oauthapplication') - op.drop_index('oauthapplication_client_id', table_name='oauthapplication') op.drop_table('oauthapplication') - op.drop_index('federatedlogin_user_id', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id_user_id', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id_service_ident', table_name='federatedlogin') - op.drop_index('federatedlogin_service_id', table_name='federatedlogin') op.drop_table('federatedlogin') - op.drop_index('buildtriggerservice_name', table_name='buildtriggerservice') op.drop_table('buildtriggerservice') - op.drop_index('user_username', table_name='user') - op.drop_index('user_stripe_id', table_name='user') - op.drop_index('user_robot', table_name='user') - op.drop_index('user_organization', table_name='user') - op.drop_index('user_email', table_name='user') op.drop_table('user') - op.drop_index('visibility_name', table_name='visibility') op.drop_table('visibility') - op.drop_index('teamrole_name', table_name='teamrole') op.drop_table('teamrole') - op.drop_index('notificationkind_name', table_name='notificationkind') op.drop_table('notificationkind') - op.drop_index('logentrykind_name', table_name='logentrykind') op.drop_table('logentrykind') - op.drop_index('role_name', table_name='role') op.drop_table('role') - op.drop_index('queueitem_queue_name', table_name='queueitem') - op.drop_index('queueitem_processing_expires', table_name='queueitem') - op.drop_index('queueitem_available_after', table_name='queueitem') - op.drop_index('queueitem_available', table_name='queueitem') op.drop_table('queueitem') op.drop_table('imagestorage') - op.drop_index('loginservice_name', table_name='loginservice') op.drop_table('loginservice') ### end Alembic commands ### diff --git a/data/migrations/versions/6f2ecf5afcf_add_the_uncompressed_size_to_image_.py b/data/migrations/versions/6f2ecf5afcf_add_the_uncompressed_size_to_image_.py new file mode 100644 index 000000000..0022ae128 --- /dev/null +++ b/data/migrations/versions/6f2ecf5afcf_add_the_uncompressed_size_to_image_.py @@ -0,0 +1,25 @@ +"""add the uncompressed size to image storage + +Revision ID: 6f2ecf5afcf +Revises: 13da56878560 +Create Date: 2014-09-22 14:39:13.470566 + +""" + +# revision identifiers, used by Alembic. +revision = '6f2ecf5afcf' +down_revision = '13da56878560' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('imagestorage', sa.Column('uncompressed_size', sa.BigInteger(), nullable=True)) + ### end Alembic commands ### + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('imagestorage', 'uncompressed_size') + ### end Alembic commands ### diff --git a/data/migrations/versions/82297d834ad_add_us_west_location.py b/data/migrations/versions/82297d834ad_add_us_west_location.py index 59eb1f800..b939a939e 100644 --- a/data/migrations/versions/82297d834ad_add_us_west_location.py +++ b/data/migrations/versions/82297d834ad_add_us_west_location.py @@ -13,24 +13,17 @@ down_revision = '47670cbeced' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models - -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - - op.bulk_insert(schema.tables['imagestoragelocation'], +def upgrade(tables): + op.bulk_insert(tables.imagestoragelocation, [ {'id':8, 'name':'s3_us_west_1'}, ]) -def downgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def downgrade(tables): op.execute( - (imagestoragelocation.delete() - .where(imagestoragelocation.c.name == op.inline_literal('s3_us_west_1'))) + (tables.imagestoragelocation.delete() + .where(tables.imagestoragelocation.c.name == op.inline_literal('s3_us_west_1'))) ) diff --git a/data/migrations/versions/9a1087b007d_allow_the_namespace_column_to_be_.py b/data/migrations/versions/9a1087b007d_allow_the_namespace_column_to_be_.py new file mode 100644 index 000000000..9b63ae190 --- /dev/null +++ b/data/migrations/versions/9a1087b007d_allow_the_namespace_column_to_be_.py @@ -0,0 +1,28 @@ +"""Allow the namespace column to be nullable. + +Revision ID: 9a1087b007d +Revises: 3f4fe1194671 +Create Date: 2014-10-01 16:11:21.277226 + +""" + +# revision identifiers, used by Alembic. +revision = '9a1087b007d' +down_revision = '3f4fe1194671' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + op.drop_index('repository_namespace_name', table_name='repository') + op.alter_column('repository', 'namespace', nullable=True, existing_type=sa.String(length=255), + server_default=sa.text('NULL')) + + +def downgrade(tables): + conn = op.get_bind() + conn.execute('update repository set namespace = (select username from user where user.id = repository.namespace_user_id) where namespace is NULL') + + op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True) + op.alter_column('repository', 'namespace', nullable=False, existing_type=sa.String(length=255)) diff --git a/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py b/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py index eda4b2840..9fc433126 100644 --- a/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py +++ b/data/migrations/versions/bcdde200a1b_add_placements_and_locations_to_the_db.py @@ -11,14 +11,10 @@ revision = 'bcdde200a1b' down_revision = '201d55b38649' from alembic import op -from data.model.sqlalchemybridge import gen_sqlalchemy_metadata -from data.database import all_models import sqlalchemy as sa -def upgrade(): - schema = gen_sqlalchemy_metadata(all_models) - +def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.create_table('imagestoragelocation', sa.Column('id', sa.Integer(), nullable=False), @@ -27,7 +23,7 @@ def upgrade(): ) op.create_index('imagestoragelocation_name', 'imagestoragelocation', ['name'], unique=True) - op.bulk_insert(schema.tables['imagestoragelocation'], + op.bulk_insert(tables.imagestoragelocation, [ {'id':1, 'name':'s3_us_east_1'}, {'id':2, 'name':'s3_eu_west_1'}, @@ -52,12 +48,8 @@ def upgrade(): ### end Alembic commands ### -def downgrade(): +def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.drop_index('imagestorageplacement_storage_id_location_id', table_name='imagestorageplacement') - op.drop_index('imagestorageplacement_storage_id', table_name='imagestorageplacement') - op.drop_index('imagestorageplacement_location_id', table_name='imagestorageplacement') op.drop_table('imagestorageplacement') - op.drop_index('imagestoragelocation_name', table_name='imagestoragelocation') op.drop_table('imagestoragelocation') ### end Alembic commands ### diff --git a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py new file mode 100644 index 000000000..9ceab4218 --- /dev/null +++ b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py @@ -0,0 +1,35 @@ +"""Remove the old webhooks table. + +Revision ID: f42b0ea7a4d +Revises: 4fdb65816b8d +Create Date: 2014-09-03 13:43:23.391464 + +""" + +# revision identifiers, used by Alembic. +revision = 'f42b0ea7a4d' +down_revision = '4fdb65816b8d' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('webhook') + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('webhook', + sa.Column('id', mysql.INTEGER(display_width=11), nullable=False), + sa.Column('public_id', mysql.VARCHAR(length=255), nullable=False), + sa.Column('repository_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=False), + sa.Column('parameters', mysql.LONGTEXT(), nullable=False), + sa.ForeignKeyConstraint(['repository_id'], [u'repository.id'], name=u'fk_webhook_repository_repository_id'), + sa.PrimaryKeyConstraint('id'), + mysql_default_charset=u'latin1', + mysql_engine=u'InnoDB' + ) + ### end Alembic commands ### diff --git a/data/model/legacy.py b/data/model/legacy.py index 9feea0738..87b4349fb 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1,12 +1,31 @@ import bcrypt import logging -import datetime import dateutil.parser import json -from data.database import * -from util.validation import * +from datetime import datetime, timedelta + +from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission, + Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, + LoginService, RepositoryBuild, Team, TeamMember, TeamRole, + LogEntryKind, LogEntry, PermissionPrototype, ImageStorage, + BuildTriggerService, RepositoryBuildTrigger, NotificationKind, + Notification, ImageStorageLocation, ImageStoragePlacement, + ExternalNotificationEvent, ExternalNotificationMethod, + RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite, + random_string_generator, db, BUILD_PHASE) +from peewee import JOIN_LEFT_OUTER, fn +from util.validation import (validate_username, validate_email, validate_password, + INVALID_PASSWORD_MESSAGE) from util.names import format_robot_username +from util.backoff import exponential_backoff + + +EXPONENTIAL_BACKOFF_SCALE = timedelta(seconds=1) +PRESUMED_DEAD_BUILD_AGE = timedelta(days=15) + + +Namespace = User.alias() logger = logging.getLogger(__name__) @@ -43,6 +62,9 @@ class InvalidRobotException(DataModelException): class InvalidTeamException(DataModelException): pass +class InvalidTeamMemberException(DataModelException): + pass + class InvalidPasswordException(DataModelException): pass @@ -64,15 +86,42 @@ class InvalidBuildTriggerException(DataModelException): pass +class InvalidImageException(DataModelException): + pass + + class TooManyUsersException(DataModelException): pass +class UserAlreadyInTeam(DataModelException): + pass + + +class TooManyLoginAttemptsException(Exception): + def __init__(self, message, retry_after): + super(TooManyLoginAttemptsException, self).__init__(message) + self.retry_after = retry_after + + +def _get_repository(namespace_name, repository_name): + return (Repository + .select(Repository, Namespace) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name) + .get()) + + +def hash_password(password, salt=None): + salt = salt or bcrypt.gensalt() + return bcrypt.hashpw(password.encode('utf-8'), salt) + + def is_create_user_allowed(): - return get_active_user_count() < config.app_config['LICENSE_USER_LIMIT'] + return True -def create_user(username, password, email): +def create_user(username, password, email, auto_verify=False): """ Creates a regular user, if allowed. """ if not validate_password(password): raise InvalidPasswordException(INVALID_PASSWORD_MESSAGE) @@ -81,15 +130,13 @@ def create_user(username, password, email): raise TooManyUsersException() created = _create_user(username, email) - - # Store the password hash - pw_hash = bcrypt.hashpw(password, bcrypt.gensalt()) - created.password_hash = pw_hash - + created.password_hash = hash_password(password) + created.verified = auto_verify created.save() return created + def _create_user(username, email): if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -181,6 +228,19 @@ def create_robot(robot_shortname, parent): except Exception as ex: raise DataModelException(ex.message) +def get_robot(robot_shortname, parent): + robot_username = format_robot_username(parent.username, robot_shortname) + robot = lookup_robot(robot_username) + + if not robot: + msg = ('Could not find robot with username: %s' % + robot_username) + raise InvalidRobotException(msg) + + service = LoginService.get(name='quayrobot') + login = FederatedLogin.get(FederatedLogin.user == robot, FederatedLogin.service == service) + + return robot, login.service_ident def lookup_robot(robot_username): joined = User.select().join(FederatedLogin).join(LoginService) @@ -191,7 +251,6 @@ def lookup_robot(robot_username): return found[0] - def verify_robot(robot_username, password): joined = User.select().join(FederatedLogin).join(LoginService) found = list(joined.where(FederatedLogin.service_ident == password, @@ -204,6 +263,25 @@ def verify_robot(robot_username, password): return found[0] +def regenerate_robot_token(robot_shortname, parent): + robot_username = format_robot_username(parent.username, robot_shortname) + + robot = lookup_robot(robot_username) + if not robot: + raise InvalidRobotException('Could not find robot with username: %s' % + robot_username) + + password = random_string_generator(length=64)() + robot.email = password + + service = LoginService.get(name='quayrobot') + login = FederatedLogin.get(FederatedLogin.user == robot, FederatedLogin.service == service) + login.service_ident = password + + login.save() + robot.save() + + return robot, password def delete_robot(robot_username): try: @@ -291,12 +369,41 @@ def remove_team(org_name, team_name, removed_by_username): team.delete_instance(recursive=True, delete_nullable=True) +def add_or_invite_to_team(inviter, team, user=None, email=None, requires_invite=True): + # If the user is a member of the organization, then we simply add the + # user directly to the team. Otherwise, an invite is created for the user/email. + # We return None if the user was directly added and the invite object if the user was invited. + if user and requires_invite: + orgname = team.organization.username + + # If the user is part of the organization (or a robot), then no invite is required. + if user.robot: + requires_invite = False + if not user.username.startswith(orgname + '+'): + raise InvalidTeamMemberException('Cannot add the specified robot to this team, ' + + 'as it is not a member of the organization') + else: + Org = User.alias() + found = User.select(User.username) + found = found.where(User.username == user.username).join(TeamMember).join(Team) + found = found.join(Org, on=(Org.username == orgname)).limit(1) + requires_invite = not any(found) + + # If we have a valid user and no invite is required, simply add the user to the team. + if user and not requires_invite: + add_user_to_team(user, team) + return None + + email_address = email if not user else None + return TeamMemberInvite.create(user=user, email=email_address, team=team, inviter=inviter) + + def add_user_to_team(user, team): try: return TeamMember.create(user=user, team=team) except Exception: - raise DataModelException('Unable to add user \'%s\' to team: \'%s\'' % - (user.username, team.name)) + raise UserAlreadyInTeam('User \'%s\' is already a member of team \'%s\'' % + (user.username, team.name)) def remove_user_from_team(org_name, team_name, username, removed_by_username): @@ -346,7 +453,8 @@ def set_team_org_permission(team, team_role_name, set_by_username): return team -def create_federated_user(username, email, service_name, service_id, set_password_notification): +def create_federated_user(username, email, service_name, service_id, + set_password_notification, metadata={}): if not is_create_user_allowed(): raise TooManyUsersException() @@ -356,7 +464,8 @@ def create_federated_user(username, email, service_name, service_id, set_passwor service = LoginService.get(LoginService.name == service_name) FederatedLogin.create(user=new_user, service=service, - service_ident=service_id) + service_ident=service_id, + metadata_json=json.dumps(metadata)) if set_password_notification: create_notification('password_required', new_user) @@ -364,9 +473,10 @@ def create_federated_user(username, email, service_name, service_id, set_passwor return new_user -def attach_federated_login(user, service_name, service_id): +def attach_federated_login(user, service_name, service_id, metadata={}): service = LoginService.get(LoginService.name == service_name) - FederatedLogin.create(user=user, service=service, service_ident=service_id) + FederatedLogin.create(user=user, service=service, service_ident=service_id, + metadata_json=json.dumps(metadata)) return user @@ -385,7 +495,7 @@ def verify_federated_login(service_name, service_id): def list_federated_logins(user): selected = FederatedLogin.select(FederatedLogin.service_ident, - LoginService.name) + LoginService.name, FederatedLogin.metadata_json) joined = selected.join(LoginService) return joined.where(LoginService.name != 'quayrobot', FederatedLogin.user == user) @@ -412,18 +522,20 @@ def confirm_user_email(code): user = code.user user.verified = True + old_email = None new_email = code.new_email if new_email: if find_user_by_email(new_email): raise DataModelException('E-mail address already used.') + old_email = user.email user.email = new_email user.save() code.delete_instance() - return user, new_email + return user, new_email, old_email def create_reset_password_email_code(email): @@ -466,6 +578,20 @@ def get_user(username): return None +def get_user_or_org(username): + try: + return User.get(User.username == username, User.robot == False) + except User.DoesNotExist: + return None + + +def get_user_by_id(user_db_id): + try: + return User.get(User.id == user_db_id, User.organization == False) + except User.DoesNotExist: + return None + + def get_user_or_org_by_customer_id(customer_id): try: return User.get(User.stripe_id == customer_id) @@ -490,12 +616,13 @@ def get_matching_users(username_prefix, robot_namespace=None, (User.robot == True))) query = (User - .select(User.username, fn.Sum(Team.id), User.robot) + .select(User.username, User.robot) .group_by(User.username) .where(direct_user_query)) if organization: query = (query + .select(User.username, User.robot, fn.Sum(Team.id)) .join(TeamMember, JOIN_LEFT_OUTER) .join(Team, JOIN_LEFT_OUTER, on=((Team.id == TeamMember.team) & (Team.organization == organization)))) @@ -504,9 +631,9 @@ def get_matching_users(username_prefix, robot_namespace=None, class MatchingUserResult(object): def __init__(self, *args): self.username = args[0] - self.is_robot = args[2] + self.is_robot = args[1] if organization: - self.is_org_member = (args[1] != None) + self.is_org_member = (args[2] != None) else: self.is_org_member = None @@ -521,11 +648,28 @@ def verify_user(username_or_email, password): except User.DoesNotExist: return None - if (fetched.password_hash and - bcrypt.hashpw(password, fetched.password_hash) == - fetched.password_hash): + now = datetime.utcnow() + + if fetched.invalid_login_attempts > 0: + can_retry_at = exponential_backoff(fetched.invalid_login_attempts, EXPONENTIAL_BACKOFF_SCALE, + fetched.last_invalid_login) + + if can_retry_at > now: + retry_after = can_retry_at - now + raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds()) + + if (fetched.password_hash and + hash_password(password, fetched.password_hash) == fetched.password_hash): + if fetched.invalid_login_attempts > 0: + fetched.invalid_login_attempts = 0 + fetched.save() + return fetched + fetched.invalid_login_attempts += 1 + fetched.last_invalid_login = now + fetched.save() + # We weren't able to authorize the user return None @@ -570,6 +714,10 @@ def get_organization_team_members(teamid): query = joined.where(Team.id == teamid) return query +def get_organization_team_member_invites(teamid): + joined = TeamMemberInvite.select().join(Team).join(User) + query = joined.where(Team.id == teamid) + return query def get_organization_member_set(orgname): Org = User.alias() @@ -600,7 +748,7 @@ def get_visible_repositories(username=None, include_public=True, page=None, limit=None, sort=False, namespace=None): query = _visible_repository_query(username=username, include_public=include_public, page=page, limit=limit, namespace=namespace, - select_models=[Repository, Visibility]) + select_models=[Repository, Namespace, Visibility]) if sort: query = query.order_by(Repository.description.desc()) @@ -614,11 +762,13 @@ def get_visible_repositories(username=None, include_public=True, page=None, def _visible_repository_query(username=None, include_public=True, limit=None, page=None, namespace=None, select_models=[]): query = (Repository - .select(*select_models) # Note: We need to leave this blank for the get_count case. Otherwise, MySQL/RDS complains. - .distinct() - .join(Visibility) - .switch(Repository) - .join(RepositoryPermission, JOIN_LEFT_OUTER)) + .select(*select_models) # MySQL/RDS complains is there are selected models for counts. + .distinct() + .join(Visibility) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(RepositoryPermission, JOIN_LEFT_OUTER)) query = _filter_to_repos_for_user(query, username, namespace, include_public) @@ -644,31 +794,25 @@ def _filter_to_repos_for_user(query, username=None, namespace=None, AdminUser = User.alias() query = (query - .switch(RepositoryPermission) - .join(User, JOIN_LEFT_OUTER) - .switch(RepositoryPermission) - .join(Team, JOIN_LEFT_OUTER) - .join(TeamMember, JOIN_LEFT_OUTER) - .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == - TeamMember.user)) - .switch(Repository) - .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace)) - .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == - AdminTeam.organization)) - .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) - .switch(AdminTeam) - .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == - AdminTeamMember.team)) - .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == - AdminUser.id))) + .switch(RepositoryPermission) + .join(User, JOIN_LEFT_OUTER) + .switch(RepositoryPermission) + .join(Team, JOIN_LEFT_OUTER) + .join(TeamMember, JOIN_LEFT_OUTER) + .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user)) + .switch(Repository) + .join(Org, JOIN_LEFT_OUTER, on=(Repository.namespace_user == Org.id)) + .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == AdminTeam.organization)) + .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) + .switch(AdminTeam) + .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == AdminTeamMember.team)) + .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == AdminUser.id))) - where_clause = ((User.username == username) | - (UserThroughTeam.username == username) | - ((AdminUser.username == username) & - (TeamRole.name == 'admin'))) + where_clause = ((User.username == username) | (UserThroughTeam.username == username) | + ((AdminUser.username == username) & (TeamRole.name == 'admin'))) if namespace: - where_clause = where_clause & (Repository.namespace == namespace) + where_clause = where_clause & (Namespace.username == namespace) if include_public: new_clause = (Visibility.name == 'public') @@ -687,7 +831,7 @@ def get_matching_repositories(repo_term, username=None): visible = get_visible_repositories(username) search_clauses = (Repository.name ** ('%' + name_term + '%') | - Repository.namespace ** ('%' + namespace_term + '%')) + Namespace.username ** ('%' + namespace_term + '%')) # Handle the case where the user has already entered a namespace path. if repo_term.find('/') > 0: @@ -696,7 +840,7 @@ def get_matching_repositories(repo_term, username=None): name_term = parts[-1] search_clauses = (Repository.name ** ('%' + name_term + '%') & - Repository.namespace ** ('%' + namespace_term + '%')) + Namespace.username ** ('%' + namespace_term + '%')) final = visible.where(search_clauses).limit(10) return list(final) @@ -706,7 +850,7 @@ def change_password(user, new_password): if not validate_password(new_password): raise InvalidPasswordException(INVALID_PASSWORD_MESSAGE) - pw_hash = bcrypt.hashpw(new_password, bcrypt.gensalt()) + pw_hash = hash_password(new_password) user.password_hash = pw_hash user.save() @@ -719,29 +863,27 @@ def change_invoice_email(user, invoice_email): user.save() -def update_email(user, new_email): +def update_email(user, new_email, auto_verify=False): user.email = new_email - user.verified = False + user.verified = auto_verify user.save() def get_all_user_permissions(user): - select = RepositoryPermission.select(RepositoryPermission, Role, Repository) - with_role = select.join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - through_user = with_repo.switch(RepositoryPermission).join(User, - JOIN_LEFT_OUTER) - as_perm = through_user.switch(RepositoryPermission) - through_team = as_perm.join(Team, JOIN_LEFT_OUTER).join(TeamMember, - JOIN_LEFT_OUTER) - UserThroughTeam = User.alias() - with_team_member = through_team.join(UserThroughTeam, JOIN_LEFT_OUTER, - on=(UserThroughTeam.id == - TeamMember.user)) - return with_team_member.where((User.id == user) | - (UserThroughTeam.id == user)) + return (RepositoryPermission + .select(RepositoryPermission, Role, Repository, Namespace) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryPermission) + .join(User, JOIN_LEFT_OUTER) + .switch(RepositoryPermission) + .join(Team, JOIN_LEFT_OUTER).join(TeamMember, JOIN_LEFT_OUTER) + .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user)) + .where((User.id == user) | (UserThroughTeam.id == user))) def delete_prototype_permission(org, uid): @@ -806,33 +948,37 @@ def get_org_wide_permissions(user): def get_all_repo_teams(namespace_name, repository_name): - select = RepositoryPermission.select(Team.name.alias('team_name'), - Role.name, RepositoryPermission) - with_team = select.join(Team) - with_role = with_team.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (RepositoryPermission.select(Team.name.alias('team_name'), Role.name, RepositoryPermission) + .join(Team) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users(namespace_name, repository_name): - select = RepositoryPermission.select(User.username, User.robot, Role.name, - RepositoryPermission) - with_user = select.join(User) - with_role = with_user.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (RepositoryPermission.select(User.username, User.robot, Role.name, RepositoryPermission) + .join(User) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users_transitive_via_teams(namespace_name, repository_name): - select = User.select().distinct() - with_team_member = select.join(TeamMember) - with_team = with_team_member.join(Team) - with_perm = with_team.join(RepositoryPermission) - with_repo = with_perm.join(Repository) - return with_repo.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + return (User + .select() + .distinct() + .join(TeamMember) + .join(Team) + .join(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def get_all_repo_users_transitive(namespace_name, repository_name): @@ -856,10 +1002,12 @@ def get_all_repo_users_transitive(namespace_name, repository_name): def get_repository_for_resource(resource_key): try: return (Repository - .select() - .join(RepositoryBuild) - .where(RepositoryBuild.resource_key == resource_key) - .get()) + .select(Repository, Namespace) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(RepositoryBuild) + .where(RepositoryBuild.resource_key == resource_key) + .get()) except Repository.DoesNotExist: return None @@ -873,8 +1021,7 @@ def lookup_repository(repo_id): def get_repository(namespace_name, repository_name): try: - return Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + return _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: return None @@ -891,11 +1038,18 @@ def get_repo_image(namespace_name, repository_name, image_id): def repository_is_public(namespace_name, repository_name): - joined = Repository.select().join(Visibility) - query = joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name, - Visibility.name == 'public') - return len(list(query)) > 0 + try: + (Repository + .select() + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Repository) + .join(Visibility) + .where(Namespace.username == namespace_name, Repository.name == repository_name, + Visibility.name == 'public') + .get()) + return True + except Repository.DoesNotExist: + return False def set_repository_visibility(repo, visibility): @@ -927,8 +1081,8 @@ def __apply_default_permissions(repo, proto_query, name_property, def create_repository(namespace, name, creating_user, visibility='private'): private = Visibility.get(name=visibility) - repo = Repository.create(namespace=namespace, name=name, - visibility=private) + namespace_user = User.get(username=namespace) + repo = Repository.create(name=name, visibility=private, namespace_user=namespace_user) admin = Role.get(name='admin') if creating_user and not creating_user.organization: @@ -994,31 +1148,34 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe def find_create_or_link_image(docker_image_id, repository, username, translations, preferred_location): with config.app_config['DB_TRANSACTION_FACTORY'](db): - repo_image = get_repo_image(repository.namespace, repository.name, + repo_image = get_repo_image(repository.namespace_user.username, repository.name, docker_image_id) if repo_image: return repo_image query = (Image - .select(Image, ImageStorage) - .distinct() - .join(ImageStorage) - .switch(Image) - .join(Repository) - .join(Visibility) - .switch(Repository) - .join(RepositoryPermission, JOIN_LEFT_OUTER)) + .select(Image, ImageStorage) + .distinct() + .join(ImageStorage) + .switch(Image) + .join(Repository) + .join(Visibility) + .switch(Repository) + .join(RepositoryPermission, JOIN_LEFT_OUTER) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(ImageStorage.uploading == False)) query = (_filter_to_repos_for_user(query, username) - .where(Image.docker_image_id == docker_image_id)) - + .where(Image.docker_image_id == docker_image_id)) + new_image_ancestry = '/' origin_image_id = None try: to_copy = query.get() msg = 'Linking image to existing storage with docker id: %s and uuid: %s' logger.debug(msg, docker_image_id, to_copy.storage.uuid) - + new_image_ancestry = __translate_ancestry(to_copy.ancestors, translations, repository, username, preferred_location) @@ -1049,49 +1206,64 @@ def find_create_or_link_image(docker_image_id, repository, username, translation return new_image -def set_image_size(docker_image_id, namespace_name, repository_name, - image_size): +def get_storage_by_uuid(storage_uuid): + placements = list(ImageStoragePlacement + .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation) + .join(ImageStorageLocation) + .switch(ImageStoragePlacement) + .join(ImageStorage) + .where(ImageStorage.uuid == storage_uuid)) + + if not placements: + raise InvalidImageException('No storage found with uuid: %s', storage_uuid) + + found = placements[0].storage + found.locations = {placement.location.name for placement in placements} + + return found + + +def set_image_size(docker_image_id, namespace_name, repository_name, image_size, uncompressed_size): try: image = (Image - .select(Image, ImageStorage) - .join(Repository) - .switch(Image) - .join(ImageStorage, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name, - Image.docker_image_id == docker_image_id) - .get()) + .select(Image, ImageStorage) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Image) + .join(ImageStorage, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + Image.docker_image_id == docker_image_id) + .get()) except Image.DoesNotExist: raise DataModelException('No image with specified id and repository') - if image.storage and image.storage.id: - image.storage.image_size = image_size - image.storage.save() - else: - image.image_size = image_size - image.save() + image.storage.image_size = image_size + image.storage.uncompressed_size = uncompressed_size + image.storage.save() return image -def set_image_metadata(docker_image_id, namespace_name, repository_name, - created_date_str, comment, command, parent=None): +def set_image_metadata(docker_image_id, namespace_name, repository_name, created_date_str, comment, + command, parent=None): with config.app_config['DB_TRANSACTION_FACTORY'](db): query = (Image - .select(Image, ImageStorage) - .join(Repository) - .switch(Image) - .join(ImageStorage) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name, - Image.docker_image_id == docker_image_id)) + .select(Image, ImageStorage) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(Image) + .join(ImageStorage) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + Image.docker_image_id == docker_image_id)) try: fetched = query.get() except Image.DoesNotExist: raise DataModelException('No image with specified id and repository') + # We cleanup any old checksum in case it's a retry after a fail + fetched.storage.checksum = None fetched.storage.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None) fetched.storage.comment = comment fetched.storage.command = command @@ -1106,14 +1278,14 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, def _get_repository_images_base(namespace_name, repository_name, query_modifier): query = (ImageStoragePlacement - .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation) - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) - .join(ImageStorage, JOIN_LEFT_OUTER) - .join(Image) - .join(Repository) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name)) + .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation) + .join(ImageStorageLocation) + .switch(ImageStoragePlacement) + .join(ImageStorage, JOIN_LEFT_OUTER) + .join(Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) query = query_modifier(query) @@ -1144,24 +1316,26 @@ def get_repository_images(namespace_name, repository_name): def list_repository_tags(namespace_name, repository_name): - select = RepositoryTag.select(RepositoryTag, Image) - with_repo = select.join(Repository) - with_image = with_repo.switch(RepositoryTag).join(Image) - return with_image.where(Repository.name == repository_name, - Repository.namespace == namespace_name) + return (RepositoryTag + .select(RepositoryTag, Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryTag) + .join(Image) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) def garbage_collect_repository(namespace_name, repository_name): with config.app_config['DB_TRANSACTION_FACTORY'](db): # Get a list of all images used by tags in the repository tag_query = (RepositoryTag - .select(RepositoryTag, Image, ImageStorage) - .join(Image) - .join(ImageStorage, JOIN_LEFT_OUTER) - .switch(RepositoryTag) - .join(Repository) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name)) + .select(RepositoryTag, Image, ImageStorage) + .join(Image) + .join(ImageStorage, JOIN_LEFT_OUTER) + .switch(RepositoryTag) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name)) referenced_anscestors = set() for tag in tag_query: @@ -1189,11 +1363,11 @@ def garbage_collect_repository(namespace_name, repository_name): if uuids_to_check_for_gc: storage_to_remove = (ImageStorage - .select() - .join(Image, JOIN_LEFT_OUTER) - .group_by(ImageStorage) - .where(ImageStorage.uuid << list(uuids_to_check_for_gc)) - .having(fn.Count(Image.id) == 0)) + .select() + .join(Image, JOIN_LEFT_OUTER) + .group_by(ImageStorage) + .where(ImageStorage.uuid << list(uuids_to_check_for_gc)) + .having(fn.Count(Image.id) == 0)) for storage in storage_to_remove: logger.debug('Garbage collecting image storage: %s', storage.uuid) @@ -1212,9 +1386,9 @@ def garbage_collect_repository(namespace_name, repository_name): def get_tag_image(namespace_name, repository_name, tag_name): def limit_to_tag(query): return (query - .switch(Image) - .join(RepositoryTag) - .where(RepositoryTag.name == tag_name)) + .switch(Image) + .join(RepositoryTag) + .where(RepositoryTag.name == tag_name)) images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag) if not images: @@ -1252,22 +1426,17 @@ def get_parent_images(namespace_name, repository_name, image_obj): def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_image_id): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: - raise DataModelException('Invalid repository %s/%s' % - (namespace_name, repository_name)) + raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) try: - image = Image.get(Image.docker_image_id == tag_docker_image_id, - Image.repository == repo) + image = Image.get(Image.docker_image_id == tag_docker_image_id, Image.repository == repo) except Image.DoesNotExist: - raise DataModelException('Invalid image with id: %s' % - tag_docker_image_id) + raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) try: - tag = RepositoryTag.get(RepositoryTag.repository == repo, - RepositoryTag.name == tag_name) + tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name) tag.image = image tag.save() except RepositoryTag.DoesNotExist: @@ -1277,41 +1446,46 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, def delete_tag(namespace_name, repository_name, tag_name): - joined = RepositoryTag.select().join(Repository) - found = list(joined.where(Repository.name == repository_name, - Repository.namespace == namespace_name, - RepositoryTag.name == tag_name)) + try: + found = (RepositoryTag + .select() + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + RepositoryTag.name == tag_name) + .get()) - if not found: + except RepositoryTag.DoesNotExist: msg = ('Invalid repository tag \'%s\' on repository \'%s/%s\'' % (tag_name, namespace_name, repository_name)) raise DataModelException(msg) - found[0].delete_instance() + found.delete_instance() def delete_all_repository_tags(namespace_name, repository_name): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: raise DataModelException('Invalid repository \'%s/%s\'' % (namespace_name, repository_name)) RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute() -def __entity_permission_repo_query(entity_id, entity_table, - entity_id_property, namespace_name, +def __entity_permission_repo_query(entity_id, entity_table, entity_id_property, namespace_name, repository_name): """ This method works for both users and teams. """ - selected = RepositoryPermission.select(entity_table, Repository, Role, - RepositoryPermission) - with_user = selected.join(entity_table) - with_role = with_user.switch(RepositoryPermission).join(Role) - with_repo = with_role.switch(RepositoryPermission).join(Repository) - return with_repo.where(Repository.name == repository_name, - Repository.namespace == namespace_name, - entity_id_property == entity_id) + + return (RepositoryPermission + .select(entity_table, Repository, Namespace, Role, RepositoryPermission) + .join(entity_table) + .switch(RepositoryPermission) + .join(Role) + .switch(RepositoryPermission) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + entity_id_property == entity_id)) def get_user_reponame_permission(username, namespace_name, repository_name): @@ -1359,8 +1533,7 @@ def delete_team_permission(team_name, namespace_name, repository_name): def __set_entity_repo_permission(entity, permission_entity_property, namespace_name, repository_name, role_name): - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) new_role = Role.get(Role.name == role_name) # Fetch any existing permission for this entity on the repo @@ -1411,15 +1584,18 @@ def purge_repository(namespace_name, repository_name): garbage_collect_repository(namespace_name, repository_name) # Delete the rest of the repository metadata - fetched = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + fetched = _get_repository(namespace_name, repository_name) fetched.delete_instance(recursive=True) def get_private_repo_count(username): - joined = Repository.select().join(Visibility) - return joined.where(Repository.namespace == username, - Visibility.name == 'private').count() + return (Repository + .select() + .join(Visibility) + .switch(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == username, Visibility.name == 'private') + .count()) def create_access_token(repository, role): @@ -1432,22 +1608,23 @@ def create_access_token(repository, role): def create_delegate_token(namespace_name, repository_name, friendly_name, role='read'): read_only = Role.get(name=role) - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) new_token = AccessToken.create(repository=repo, role=read_only, friendly_name=friendly_name, temporary=False) return new_token def get_repository_delegate_tokens(namespace_name, repository_name): - return (AccessToken.select(AccessToken, Role) - .join(Repository) - .switch(AccessToken) - .join(Role) - .switch(AccessToken) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, Repository.namespace == namespace_name, - AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None)) + return (AccessToken + .select(AccessToken, Role) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(AccessToken) + .join(Role) + .switch(AccessToken) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + AccessToken.temporary == False, RepositoryBuildTrigger.uuid >> None)) def get_repo_delegate_token(namespace_name, repository_name, code): @@ -1481,14 +1658,17 @@ def delete_delegate_token(namespace_name, repository_name, code): def load_token_data(code): """ Load the permissions for any token by code. """ - selected = AccessToken.select(AccessToken, Repository, Role) - with_role = selected.join(Role) - with_repo = with_role.switch(AccessToken).join(Repository) - fetched = list(with_repo.where(AccessToken.code == code)) + try: + return (AccessToken + .select(AccessToken, Repository, Namespace, Role) + .join(Role) + .switch(AccessToken) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(AccessToken.code == code) + .get()) - if fetched: - return fetched[0] - else: + except AccessToken.DoesNotExist: raise InvalidTokenException('Invalid delegate token code: %s' % code) @@ -1505,15 +1685,15 @@ def get_repository_build(namespace_name, repository_name, build_uuid): def list_repository_builds(namespace_name, repository_name, limit, include_inactive=True): query = (RepositoryBuild - .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) - .join(Repository) - .switch(RepositoryBuild) - .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) - .join(BuildTriggerService, JOIN_LEFT_OUTER) - .where(Repository.name == repository_name, - Repository.namespace == namespace_name) - .order_by(RepositoryBuild.started.desc()) - .limit(limit)) + .select(RepositoryBuild, RepositoryBuildTrigger, BuildTriggerService) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuild) + .join(RepositoryBuildTrigger, JOIN_LEFT_OUTER) + .join(BuildTriggerService, JOIN_LEFT_OUTER) + .where(Repository.name == repository_name, Namespace.username == namespace_name) + .order_by(RepositoryBuild.started.desc()) + .limit(limit)) if not include_inactive: query = query.where(RepositoryBuild.phase != 'error', @@ -1577,16 +1757,17 @@ def create_repo_notification(repo, event_name, method_name, config): def get_repo_notification(namespace_name, repository_name, uuid): - joined = RepositoryNotification.select().join(Repository) - found = list(joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name, - RepositoryNotification.uuid == uuid)) - - if not found: + try: + return (RepositoryNotification + .select(RepositoryNotification, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name, + RepositoryNotification.uuid == uuid) + .get()) + except RepositoryNotification.DoesNotExist: raise InvalidNotificationException('No repository notification found with id: %s' % uuid) - return found[0] - def delete_repo_notification(namespace_name, repository_name, uuid): found = get_repo_notification(namespace_name, repository_name, uuid) @@ -1595,15 +1776,19 @@ def delete_repo_notification(namespace_name, repository_name, uuid): def list_repo_notifications(namespace_name, repository_name, event_name=None): - joined = RepositoryNotification.select().join(Repository) - where = joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + query = (RepositoryNotification + .select(RepositoryNotification, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) if event_name: - event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name) - where = where.where(RepositoryNotification.event == event) + query = (query + .switch(RepositoryNotification) + .join(ExternalNotificationEvent) + .where(ExternalNotificationEvent.name == event_name)) - return where + return query def list_logs(start_time, end_time, performer=None, repository=None, namespace=None): @@ -1647,16 +1832,17 @@ def create_build_trigger(repo, service_name, auth_token, user, pull_robot=None): def get_build_trigger(namespace_name, repository_name, trigger_uuid): try: return (RepositoryBuildTrigger - .select(RepositoryBuildTrigger, BuildTriggerService, Repository) - .join(BuildTriggerService) - .switch(RepositoryBuildTrigger) - .join(Repository) - .switch(RepositoryBuildTrigger) - .join(User) - .where(RepositoryBuildTrigger.uuid == trigger_uuid, - Repository.namespace == namespace_name, - Repository.name == repository_name) - .get()) + .select(RepositoryBuildTrigger, BuildTriggerService, Repository, Namespace) + .join(BuildTriggerService) + .switch(RepositoryBuildTrigger) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryBuildTrigger) + .join(User) + .where(RepositoryBuildTrigger.uuid == trigger_uuid, + Namespace.username == namespace_name, + Repository.name == repository_name) + .get()) except RepositoryBuildTrigger.DoesNotExist: msg = 'No build trigger with uuid: %s' % trigger_uuid raise InvalidBuildTriggerException(msg) @@ -1664,12 +1850,12 @@ def get_build_trigger(namespace_name, repository_name, trigger_uuid): def list_build_triggers(namespace_name, repository_name): return (RepositoryBuildTrigger - .select(RepositoryBuildTrigger, BuildTriggerService, Repository) - .join(BuildTriggerService) - .switch(RepositoryBuildTrigger) - .join(Repository) - .where(Repository.namespace == namespace_name, - Repository.name == repository_name)) + .select(RepositoryBuildTrigger, BuildTriggerService, Repository) + .join(BuildTriggerService) + .switch(RepositoryBuildTrigger) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace_name, Repository.name == repository_name)) def list_trigger_builds(namespace_name, repository_name, trigger_uuid, @@ -1687,19 +1873,20 @@ def create_notification(kind_name, target, metadata={}): def create_unique_notification(kind_name, target, metadata={}): with config.app_config['DB_TRANSACTION_FACTORY'](db): - if list_notifications(target, kind_name).count() == 0: + if list_notifications(target, kind_name, limit=1).count() == 0: create_notification(kind_name, target, metadata) def lookup_notification(user, uuid): - results = list(list_notifications(user, id_filter=uuid, include_dismissed=True)) + results = list(list_notifications(user, id_filter=uuid, include_dismissed=True, limit=1)) if not results: return None return results[0] -def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=False): +def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=False, + page=None, limit=None): Org = User.alias() AdminTeam = Team.alias() AdminTeamMember = TeamMember.alias() @@ -1737,6 +1924,11 @@ def list_notifications(user, kind_name=None, id_filter=None, include_dismissed=F .switch(Notification) .where(Notification.uuid == id_filter)) + if page: + query = query.paginate(page, limit) + elif limit: + query = query.limit(limit) + return query @@ -1753,17 +1945,57 @@ def delete_notifications_by_kind(target, kind_name): Notification.kind == kind_ref).execute() +def delete_matching_notifications(target, kind_name, **kwargs): + kind_ref = NotificationKind.get(name=kind_name) + + # Load all notifications for the user with the given kind. + notifications = Notification.select().where( + Notification.target == target, + Notification.kind == kind_ref) + + # For each, match the metadata to the specified values. + for notification in notifications: + matches = True + try: + metadata = json.loads(notification.metadata_json) + except: + continue + + for (key, value) in kwargs.iteritems(): + if not key in metadata or metadata[key] != value: + matches = False + break + + if not matches: + continue + + notification.delete_instance() + + def get_active_users(): return User.select().where(User.organization == False, User.robot == False) + def get_active_user_count(): return get_active_users().count() + +def detach_external_login(user, service_name): + try: + service = LoginService.get(name = service_name) + except LoginService.DoesNotExist: + return + + FederatedLogin.delete().where(FederatedLogin.user == user, + FederatedLogin.service == service).execute() + + def delete_user(user): user.delete_instance(recursive=True, delete_nullable=True) # TODO: also delete any repository data associated + def check_health(): # We will connect to the db, check that it contains some log entry kinds try: @@ -1772,24 +2004,23 @@ def check_health(): except: return False -def get_email_authorized_for_repo(namespace, repository, email): - found = list(RepositoryAuthorizedEmail.select() - .join(Repository) - .where(Repository.namespace == namespace, - Repository.name == repository, - RepositoryAuthorizedEmail.email == email) - .switch(RepositoryAuthorizedEmail) - .limit(1)) - if not found or len(found) < 1: - return None - return found[0] +def get_email_authorized_for_repo(namespace, repository, email): + try: + return (RepositoryAuthorizedEmail + .select(RepositoryAuthorizedEmail, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Namespace.username == namespace, Repository.name == repository, + RepositoryAuthorizedEmail.email == email) + .get()) + except RepositoryAuthorizedEmail.DoesNotExist: + return None def create_email_authorization_for_repo(namespace_name, repository_name, email): try: - repo = Repository.get(Repository.name == repository_name, - Repository.namespace == namespace_name) + repo = _get_repository(namespace_name, repository_name) except Repository.DoesNotExist: raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name)) @@ -1799,7 +2030,12 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email): def confirm_email_authorization_for_repo(code): try: - found = RepositoryAuthorizedEmail.get(RepositoryAuthorizedEmail.code == code) + found = (RepositoryAuthorizedEmail + .select(RepositoryAuthorizedEmail, Repository, Namespace) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(RepositoryAuthorizedEmail.code == code) + .get()) except RepositoryAuthorizedEmail.DoesNotExist: raise DataModelException('Invalid confirmation code.') @@ -1807,3 +2043,72 @@ def confirm_email_authorization_for_repo(code): found.save() return found + + +def delete_team_email_invite(team, email): + found = TeamMemberInvite.get(TeamMemberInvite.email == email, TeamMemberInvite.team == team) + found.delete_instance() + +def delete_team_user_invite(team, user): + try: + found = TeamMemberInvite.get(TeamMemberInvite.user == user, TeamMemberInvite.team == team) + except TeamMemberInvite.DoesNotExist: + return False + + found.delete_instance() + return True + +def lookup_team_invites(user): + return TeamMemberInvite.select().where(TeamMemberInvite.user == user) + +def lookup_team_invite(code, user=None): + # Lookup the invite code. + try: + found = TeamMemberInvite.get(TeamMemberInvite.invite_token == code) + except TeamMemberInvite.DoesNotExist: + raise DataModelException('Invalid confirmation code.') + + if user and found.user != user: + raise DataModelException('Invalid confirmation code.') + + return found + +def delete_team_invite(code, user=None): + found = lookup_team_invite(code, user) + + team = found.team + inviter = found.inviter + + found.delete_instance() + + return (team, inviter) + + +def confirm_team_invite(code, user): + found = lookup_team_invite(code) + + # If the invite is for a specific user, we have to confirm that here. + if found.user is not None and found.user != user: + message = """This invite is intended for user "%s". + Please login to that account and try again.""" % found.user.username + raise DataModelException(message) + + # Add the user to the team. + try: + add_user_to_team(user, found.team) + except UserAlreadyInTeam: + # Ignore. + pass + + # Delete the invite and return the team. + team = found.team + inviter = found.inviter + found.delete_instance() + return (team, inviter) + +def archivable_buildlogs_query(): + presumed_dead_date = datetime.utcnow() - PRESUMED_DEAD_BUILD_AGE + return (RepositoryBuild.select() + .where((RepositoryBuild.phase == BUILD_PHASE.COMPLETE) | + (RepositoryBuild.phase == BUILD_PHASE.ERROR) | + (RepositoryBuild.started < presumed_dead_date), RepositoryBuild.logs_archived == False)) diff --git a/data/model/oauth.py b/data/model/oauth.py index 309e2122a..51bfc053e 100644 --- a/data/model/oauth.py +++ b/data/model/oauth.py @@ -46,7 +46,7 @@ class DatabaseAuthorizationProvider(AuthorizationProvider): def validate_redirect_uri(self, client_id, redirect_uri): try: app = OAuthApplication.get(client_id=client_id) - if app.redirect_uri and redirect_uri.startswith(app.redirect_uri): + if app.redirect_uri and redirect_uri and redirect_uri.startswith(app.redirect_uri): return True return False except OAuthApplication.DoesNotExist: diff --git a/data/model/sqlalchemybridge.py b/data/model/sqlalchemybridge.py index 46809fb21..8b7d8b664 100644 --- a/data/model/sqlalchemybridge.py +++ b/data/model/sqlalchemybridge.py @@ -17,7 +17,12 @@ OPTION_TRANSLATIONS = { def gen_sqlalchemy_metadata(peewee_model_list): - metadata = MetaData() + metadata = MetaData(naming_convention={ + "ix": 'ix_%(column_0_label)s', + "uq": "uq_%(table_name)s_%(column_0_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s" + }) for model in peewee_model_list: meta = model._meta diff --git a/data/queue.py b/data/queue.py index 44d7ad531..79e645ebf 100644 --- a/data/queue.py +++ b/data/queue.py @@ -68,9 +68,8 @@ class WorkQueue(object): 'retries_remaining': retries_remaining, } - if available_after: - available_date = datetime.utcnow() + timedelta(seconds=available_after) - params['available_after'] = available_date + available_date = datetime.utcnow() + timedelta(seconds=available_after or 0) + params['available_after'] = available_date with self._transaction_factory(db): QueueItem.create(**params) diff --git a/data/userevent.py b/data/userevent.py index bcdafd078..b45d4e4fa 100644 --- a/data/userevent.py +++ b/data/userevent.py @@ -7,14 +7,14 @@ class UserEventBuilder(object): Defines a helper class for constructing UserEvent and UserEventListener instances. """ - def __init__(self, redis_host): - self._redis_host = redis_host + def __init__(self, redis_config): + self._redis_config = redis_config def get_event(self, username): - return UserEvent(self._redis_host, username) + return UserEvent(self._redis_config, username) def get_listener(self, username, events): - return UserEventListener(self._redis_host, username, events) + return UserEventListener(self._redis_config, username, events) class UserEventsBuilderModule(object): @@ -26,8 +26,14 @@ class UserEventsBuilderModule(object): self.state = None def init_app(self, app): - redis_hostname = app.config.get('USER_EVENTS_REDIS_HOSTNAME') - user_events = UserEventBuilder(redis_hostname) + redis_config = app.config.get('USER_EVENTS_REDIS') + if not redis_config: + # This is the old key name. + redis_config = { + 'host': app.config.get('USER_EVENTS_REDIS_HOSTNAME') + } + + user_events = UserEventBuilder(redis_config) # register extension with app app.extensions = getattr(app, 'extensions', {}) @@ -43,8 +49,8 @@ class UserEvent(object): Defines a helper class for publishing to realtime user events as backed by Redis. """ - def __init__(self, redis_host, username): - self._redis = redis.StrictRedis(host=redis_host) + def __init__(self, redis_config, username): + self._redis = redis.StrictRedis(socket_connect_timeout=5, **redis_config) self._username = username @staticmethod @@ -74,10 +80,10 @@ class UserEventListener(object): Defines a helper class for subscribing to realtime user events as backed by Redis. """ - def __init__(self, redis_host, username, events=set([])): + def __init__(self, redis_config, username, events=set([])): channels = [self._user_event_key(username, e) for e in events] - self._redis = redis.StrictRedis(host=redis_host) + self._redis = redis.StrictRedis(socket_connect_timeout=5, **redis_config) self._pubsub = self._redis.pubsub() self._pubsub.subscribe(channels) diff --git a/data/userfiles.py b/data/userfiles.py index 79fbcb507..f4b786df5 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -1,110 +1,35 @@ -import boto import os import logging -import hashlib import magic -from boto.s3.key import Key from uuid import uuid4 from flask import url_for, request, send_file, make_response, abort from flask.views import View +from _pyio import BufferedReader logger = logging.getLogger(__name__) -class FakeUserfiles(object): - def prepare_for_drop(self, mime_type): - return ('http://fake/url', uuid4()) - - def store_file(self, file_like_obj, content_type): - raise NotImplementedError() - - def get_file_url(self, file_id, expires_in=300): - return ('http://fake/url') - - def get_file_checksum(self, file_id): - return 'abcdefg' - - -class S3FileWriteException(Exception): - pass - - -class S3Userfiles(object): - def __init__(self, path, s3_access_key, s3_secret_key, bucket_name): - self._initialized = False - self._bucket_name = bucket_name - self._access_key = s3_access_key - self._secret_key = s3_secret_key - self._prefix = path - self._s3_conn = None - self._bucket = None - - def _initialize_s3(self): - if not self._initialized: - self._s3_conn = boto.connect_s3(self._access_key, self._secret_key) - self._bucket = self._s3_conn.get_bucket(self._bucket_name) - self._initialized = True - - def prepare_for_drop(self, mime_type): - """ Returns a signed URL to upload a file to our bucket. """ - self._initialize_s3() - logger.debug('Requested upload url with content type: %s' % mime_type) - file_id = str(uuid4()) - full_key = os.path.join(self._prefix, file_id) - k = Key(self._bucket, full_key) - url = k.generate_url(300, 'PUT', headers={'Content-Type': mime_type}, - encrypt_key=True) - return (url, file_id) - - def store_file(self, file_like_obj, content_type): - self._initialize_s3() - file_id = str(uuid4()) - full_key = os.path.join(self._prefix, file_id) - k = Key(self._bucket, full_key) - logger.debug('Setting s3 content type to: %s' % content_type) - k.set_metadata('Content-Type', content_type) - bytes_written = k.set_contents_from_file(file_like_obj, encrypt_key=True, - rewind=True) - - if bytes_written == 0: - raise S3FileWriteException('Unable to write file to S3') - - return file_id - - def get_file_url(self, file_id, expires_in=300, mime_type=None): - self._initialize_s3() - full_key = os.path.join(self._prefix, file_id) - k = Key(self._bucket, full_key) - headers = None - if mime_type: - headers={'Content-Type': mime_type} - - return k.generate_url(expires_in, headers=headers) - - def get_file_checksum(self, file_id): - self._initialize_s3() - full_key = os.path.join(self._prefix, file_id) - k = self._bucket.lookup(full_key) - return k.etag[1:-1][:7] - - class UserfilesHandlers(View): methods = ['GET', 'PUT'] - def __init__(self, local_userfiles): - self._userfiles = local_userfiles + def __init__(self, distributed_storage, location, files): + self._storage = distributed_storage + self._files = files + self._locations = {location} self._magic = magic.Magic(mime=True) def get(self, file_id): - path = self._userfiles.file_path(file_id) - if not os.path.exists(path): + path = self._files.get_file_id_path(file_id) + try: + file_stream = self._storage.stream_read_file(self._locations, path) + buffered = BufferedReader(file_stream) + file_header_bytes = buffered.peek(1024) + return send_file(buffered, mimetype=self._magic.from_buffer(file_header_bytes)) + except IOError: abort(404) - logger.debug('Sending path: %s' % path) - return send_file(path, mimetype=self._magic.from_file(path)) - def put(self, file_id): input_stream = request.stream if request.headers.get('transfer-encoding') == 'chunked': @@ -112,7 +37,10 @@ class UserfilesHandlers(View): # encoding (Gunicorn) input_stream = request.environ['wsgi.input'] - self._userfiles.store_stream(input_stream, file_id) + c_type = request.headers.get('Content-Type', None) + + path = self._files.get_file_id_path(file_id) + self._storage.stream_write(self._locations, path, input_stream, c_type) return make_response('Okay') @@ -123,99 +51,82 @@ class UserfilesHandlers(View): return self.put(file_id) -class LocalUserfiles(object): - def __init__(self, app, path): - self._root_path = path - self._buffer_size = 64 * 1024 # 64 KB +class DelegateUserfiles(object): + def __init__(self, app, distributed_storage, location, path, handler_name): self._app = app + self._storage = distributed_storage + self._locations = {location} + self._prefix = path + self._handler_name = handler_name def _build_url_adapter(self): return self._app.url_map.bind(self._app.config['SERVER_HOSTNAME'], script_name=self._app.config['APPLICATION_ROOT'] or '/', url_scheme=self._app.config['PREFERRED_URL_SCHEME']) - def prepare_for_drop(self, mime_type): + def get_file_id_path(self, file_id): + return os.path.join(self._prefix, file_id) + + def prepare_for_drop(self, mime_type, requires_cors=True): + """ Returns a signed URL to upload a file to our bucket. """ + logger.debug('Requested upload url with content type: %s' % mime_type) file_id = str(uuid4()) - with self._app.app_context() as ctx: - ctx.url_adapter = self._build_url_adapter() - return (url_for('userfiles_handlers', file_id=file_id, _external=True), file_id) + path = self.get_file_id_path(file_id) + url = self._storage.get_direct_upload_url(self._locations, path, mime_type, requires_cors) - def file_path(self, file_id): - if '..' in file_id or file_id.startswith('/'): - raise RuntimeError('Invalid Filename') - return os.path.join(self._root_path, file_id) + if url is None: + with self._app.app_context() as ctx: + ctx.url_adapter = self._build_url_adapter() + return (url_for(self._handler_name, file_id=file_id, _external=True), file_id) - def store_stream(self, stream, file_id): - path = self.file_path(file_id) - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - os.makedirs(dirname) + return (url, file_id) - with open(path, 'w') as to_write: - while True: - try: - buf = stream.read(self._buffer_size) - if not buf: - break - to_write.write(buf) - except IOError: - break + def store_file(self, file_like_obj, content_type, content_encoding=None, file_id=None): + if file_id is None: + file_id = str(uuid4()) - def store_file(self, file_like_obj, content_type): - file_id = str(uuid4()) - - # Rewind the file to match what s3 does - file_like_obj.seek(0, os.SEEK_SET) - - self.store_stream(file_like_obj, file_id) + path = self.get_file_id_path(file_id) + self._storage.stream_write(self._locations, path, file_like_obj, content_type, + content_encoding) return file_id - def get_file_url(self, file_id, expires_in=300): - with self._app.app_context() as ctx: - ctx.url_adapter = self._build_url_adapter() - return url_for('userfiles_handlers', file_id=file_id, _external=True) + def get_file_url(self, file_id, expires_in=300, requires_cors=False): + path = self.get_file_id_path(file_id) + url = self._storage.get_direct_download_url(self._locations, path, expires_in, requires_cors) + + if url is None: + with self._app.app_context() as ctx: + ctx.url_adapter = self._build_url_adapter() + return url_for(self._handler_name, file_id=file_id, _external=True) + + return url def get_file_checksum(self, file_id): - path = self.file_path(file_id) - sha_hash = hashlib.sha256() - with open(path, 'r') as to_hash: - while True: - buf = to_hash.read(self._buffer_size) - if not buf: - break - sha_hash.update(buf) - return sha_hash.hexdigest()[:7] + path = self.get_file_id_path(file_id) + return self._storage.get_checksum(self._locations, path) class Userfiles(object): - def __init__(self, app=None): + def __init__(self, app=None, distributed_storage=None): self.app = app if app is not None: - self.state = self.init_app(app) + self.state = self.init_app(app, distributed_storage) else: self.state = None - def init_app(self, app): - storage_type = app.config.get('USERFILES_TYPE', 'LocalUserfiles') - path = app.config.get('USERFILES_PATH', '') + def init_app(self, app, distributed_storage): + location = app.config.get('USERFILES_LOCATION') + path = app.config.get('USERFILES_PATH', None) - if storage_type == 'LocalUserfiles': - userfiles = LocalUserfiles(app, path) - app.add_url_rule('/userfiles/', - view_func=UserfilesHandlers.as_view('userfiles_handlers', - local_userfiles=userfiles)) + handler_name = 'userfiles_handlers' - elif storage_type == 'S3Userfiles': - access_key = app.config.get('USERFILES_AWS_ACCESS_KEY', '') - secret_key = app.config.get('USERFILES_AWS_SECRET_KEY', '') - bucket = app.config.get('USERFILES_S3_BUCKET', '') - userfiles = S3Userfiles(path, access_key, secret_key, bucket) + userfiles = DelegateUserfiles(app, distributed_storage, location, path, handler_name) - elif storage_type == 'FakeUserfiles': - userfiles = FakeUserfiles() - - else: - raise RuntimeError('Unknown userfiles type: %s' % storage_type) + app.add_url_rule('/userfiles/', + view_func=UserfilesHandlers.as_view(handler_name, + distributed_storage=distributed_storage, + location=location, + files=userfiles)) # register extension with app app.extensions = getattr(app, 'extensions', {}) diff --git a/emails/base.html b/emails/base.html new file mode 100644 index 000000000..33dac53de --- /dev/null +++ b/emails/base.html @@ -0,0 +1,45 @@ + + + + + + {{ subject }} + + + + + + + +
+ +
+ +
{{ app_title }}
+ +
+ + +
+ +
+ +
+ {% block content %}{% endblock %} +
+ +
+ + diff --git a/emails/changeemail.html b/emails/changeemail.html new file mode 100644 index 000000000..ee9b909fc --- /dev/null +++ b/emails/changeemail.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} + +

E-mail Address Change Requested

+ +This email address was recently asked to become the new e-mail address for user {{ username | user_reference }}. +
+
+To confirm this change, please click the following link:
+{{ app_link('confirm?code=' + token) }} + +{% endblock %} diff --git a/emails/confirmemail.html b/emails/confirmemail.html new file mode 100644 index 000000000..de94372cd --- /dev/null +++ b/emails/confirmemail.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} + +

Please Confirm E-mail Address

+ +This email address was recently used to register user {{ username | user_reference }}. +
+
+To confirm this email address, please click the following link:
+{{ app_link('confirm?code=' + token) }} + +{% endblock %} diff --git a/emails/emailchanged.html b/emails/emailchanged.html new file mode 100644 index 000000000..ce6de5565 --- /dev/null +++ b/emails/emailchanged.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} + +{% block content %} + +

Account E-mail Address Changed

+ +The email address for user {{ username | user_reference }} has been changed from this e-mail address to {{ new_email }}. +
+
+If this change was not expected, please immediately log into your {{ username | admin_reference }} and reset your email address. + +{% endblock %} diff --git a/emails/passwordchanged.html b/emails/passwordchanged.html new file mode 100644 index 000000000..07c6232cc --- /dev/null +++ b/emails/passwordchanged.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} + +

Account Password Changed

+ +The password for user {{ username | user_reference }} has been updated. +
+
+If this change was not expected, please immediately log into your account settings and reset your email address, +or contact support. + +{% endblock %} diff --git a/emails/paymentfailure.html b/emails/paymentfailure.html new file mode 100644 index 000000000..790f590b4 --- /dev/null +++ b/emails/paymentfailure.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} + +

Subscription Payment Failure

+ +Your recent payment for account {{ username | user_reference }} failed, which usually results in our payments processor canceling +your subscription automatically. If you would like to continue to use {{ app_title }} without interruption, +please add a new card to {{ app_title }} and re-subscribe to your plan.
+
+You can find the card and subscription management features under your {{ username | admin_reference }}
+ +{% endblock %} diff --git a/emails/recovery.html b/emails/recovery.html new file mode 100644 index 000000000..6f0267e39 --- /dev/null +++ b/emails/recovery.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block content %} + +

Account recovery

+ +A user at {{ app_link() }} has attempted to recover their account +using this email address. +
+
+If you made this request, please click the following link to recover your account and +change your password: +{{ app_link('recovery?code=' + token) }} +

+If you did not make this request, your account has not been compromised and the user was +not given access. Please disregard this email. + +{% endblock %} diff --git a/emails/repoauthorizeemail.html b/emails/repoauthorizeemail.html new file mode 100644 index 000000000..7ae33975c --- /dev/null +++ b/emails/repoauthorizeemail.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} + +

Verify e-mail to receive repository notifications

+ +A request has been made to send notifications to this email address for repository {{ (namespace, repository) | repository_reference }} + +

+To verify this email address, please click the following link:
+{{ app_link('authrepoemail?code=' + token) }} + +{% endblock %} diff --git a/emails/teaminvite.html b/emails/teaminvite.html new file mode 100644 index 000000000..3d8ff9c14 --- /dev/null +++ b/emails/teaminvite.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} + +{% block content %} + +

Invitation to join team: {{ teamname }}

+ +{{ inviter | user_reference }} has invited you to join the team {{ teamname }} under organization {{ organization | user_reference }}. + +

+ +To join the team, please click the following link:
+{{ app_link('confirminvite?code=' + token) }} + +

+If you were not expecting this invitation, you can ignore this email. + +{% endblock %} diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 0234e6820..a885c10b7 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -1,8 +1,9 @@ import logging import json +import datetime from app import app -from flask import Blueprint, request, make_response, jsonify +from flask import Blueprint, request, make_response, jsonify, session from flask.ext.restful import Resource, abort, Api, reqparse from flask.ext.restful.utils.cors import crossdomain from werkzeug.exceptions import HTTPException @@ -53,11 +54,6 @@ class InvalidRequest(ApiException): ApiException.__init__(self, 'invalid_request', 400, error_description, payload) -class InvalidResponse(ApiException): - def __init__(self, error_description, payload=None): - ApiException.__init__(self, 'invalid_response', 500, error_description, payload) - - class InvalidToken(ApiException): def __init__(self, error_description, payload=None): ApiException.__init__(self, 'invalid_token', 401, error_description, payload) @@ -72,6 +68,11 @@ class Unauthorized(ApiException): ApiException.__init__(self, 'insufficient_scope', 403, 'Unauthorized', payload) +class FreshLoginRequired(ApiException): + def __init__(self, payload=None): + ApiException.__init__(self, 'fresh_login_required', 401, "Requires fresh login", payload) + + class ExceedsLicenseException(ApiException): def __init__(self, payload=None): ApiException.__init__(self, None, 402, 'Payment Required', payload) @@ -93,6 +94,14 @@ def handle_api_error(error): return response +@api_bp.app_errorhandler(model.TooManyLoginAttemptsException) +@crossdomain(origin='*', headers=['Authorization', 'Content-Type']) +def handle_too_many_login_attempts(error): + response = make_response('Too many login attempts', 429) + response.headers['Retry-After'] = int(error.retry_after) + return response + + def resource(*urls, **kwargs): def wrapper(api_resource): if not api_resource: @@ -163,7 +172,7 @@ def path_param(name, description): def add_param(func): if not func: return func - + if '__api_path_params' not in dir(func): func.__api_path_params = {} func.__api_path_params[name] = { @@ -265,6 +274,26 @@ def require_user_permission(permission_class, scope=None): require_user_read = require_user_permission(UserReadPermission, scopes.READ_USER) require_user_admin = require_user_permission(UserAdminPermission, None) +require_fresh_user_admin = require_user_permission(UserAdminPermission, None) + +def require_fresh_login(func): + @add_method_metadata('requires_fresh_login', True) + @wraps(func) + def wrapped(*args, **kwargs): + user = get_authenticated_user() + if not user: + raise Unauthorized() + + logger.debug('Checking fresh login for user %s', user.username) + + last_login = session.get('login_time', datetime.datetime.min) + valid_span = datetime.datetime.now() - datetime.timedelta(minutes=10) + + if not user.password_hash or last_login >= valid_span: + return func(*args, **kwargs) + + raise FreshLoginRequired() + return wrapped def require_scope(scope_object): @@ -292,25 +321,6 @@ def validate_json_request(schema_name): return wrapper -def define_json_response(schema_name): - def wrapper(func): - @add_method_metadata('response_schema', schema_name) - @wraps(func) - def wrapped(self, *args, **kwargs): - schema = self.schemas[schema_name] - try: - resp = func(self, *args, **kwargs) - - if app.config['TESTING']: - validate(resp, schema) - - return resp - except ValidationError as ex: - raise InvalidResponse(ex.message) - return wrapped - return wrapper - - def request_error(exception=None, **kwargs): data = kwargs.copy() message = 'Request error.' @@ -338,6 +348,25 @@ def log_action(kind, user_or_orgname, metadata=None, repo=None): metadata=metadata, repository=repo) +def define_json_response(schema_name): + def wrapper(func): + @add_method_metadata('response_schema', schema_name) + @wraps(func) + def wrapped(self, *args, **kwargs): + schema = self.schemas[schema_name] + resp = func(self, *args, **kwargs) + + if app.config['TESTING']: + try: + validate(resp, schema) + except ValidationError as ex: + raise InvalidResponse(ex.message) + + return resp + return wrapped + return wrapper + + import endpoints.api.billing import endpoints.api.build import endpoints.api.discovery diff --git a/endpoints/api/billing.py b/endpoints/api/billing.py index f5b022ca6..de3c2126a 100644 --- a/endpoints/api/billing.py +++ b/endpoints/api/billing.py @@ -4,7 +4,7 @@ from flask import request from app import billing from endpoints.api import (resource, nickname, ApiResource, validate_json_request, log_action, related_user_resource, internal_only, Unauthorized, NotFound, - require_user_admin, show_if, hide_if, path_param, require_scope) + require_user_admin, show_if, hide_if, path_param, require_scope, abort) from endpoints.api.subscribe import subscribe, subscription_view from auth.permissions import AdministerOrganizationPermission from auth.auth_context import get_authenticated_user @@ -24,7 +24,11 @@ def get_card(user): } if user.stripe_id: - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') + if cus and cus.default_card: # Find the default card. default_card = None @@ -47,7 +51,11 @@ def get_card(user): def set_card(user, token): if user.stripe_id: - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') + if cus: try: cus.card = token @@ -56,6 +64,8 @@ def set_card(user, token): return carderror_response(exc) except stripe.InvalidRequestError as exc: return carderror_response(exc) + except stripe.APIConnectionError as e: + return carderror_response(e) return get_card(user) @@ -76,7 +86,11 @@ def get_invoices(customer_id): 'plan': i.lines.data[0].plan.id if i.lines.data[0].plan else None } - invoices = billing.Invoice.all(customer=customer_id, count=12) + try: + invoices = billing.Invoice.all(customer=customer_id, count=12) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') + return { 'invoices': [invoice_view(i) for i in invoices.data] } @@ -231,7 +245,10 @@ class UserPlan(ApiResource): private_repos = model.get_private_repo_count(user.username) if user.stripe_id: - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') if cus.subscription: return subscription_view(cus.subscription, private_repos) @@ -297,7 +314,10 @@ class OrganizationPlan(ApiResource): private_repos = model.get_private_repo_count(orgname) organization = model.get_organization(orgname) if organization.stripe_id: - cus = billing.Customer.retrieve(organization.stripe_id) + try: + cus = billing.Customer.retrieve(organization.stripe_id) + except stripe.APIConnectionError as e: + abort(503, message='Cannot contact Stripe') if cus.subscription: return subscription_view(cus.subscription, private_repos) diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 7fa11cc15..e416b0083 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -1,9 +1,9 @@ import logging import json -from flask import request +from flask import request, redirect -from app import app, userfiles as user_files, build_logs +from app import app, userfiles as user_files, build_logs, log_archive from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, ApiResource, internal_only, format_date, api, Unauthorized, NotFound, @@ -81,7 +81,7 @@ def build_status_view(build_obj, can_write=False): } if can_write: - resp['archive_url'] = user_files.get_file_url(build_obj.resource_key) + resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True) return resp @@ -171,7 +171,7 @@ class RepositoryBuildList(RepositoryParamResource): # was used. associated_repository = model.get_repository_for_resource(dockerfile_id) if associated_repository: - if not ModifyRepositoryPermission(associated_repository.namespace, + if not ModifyRepositoryPermission(associated_repository.namespace_user.username, associated_repository.name): raise Unauthorized() @@ -221,6 +221,10 @@ class RepositoryBuildLogs(RepositoryParamResource): build = model.get_repository_build(namespace, repository, build_uuid) + # If the logs have been archived, just redirect to the completed archive + if build.logs_archived: + return redirect(log_archive.get_file_url(build.uuid)) + start = int(request.args.get('start', 0)) try: @@ -263,7 +267,7 @@ class FileDropResource(ApiResource): def post(self): """ Request a URL to which a file may be uploaded. """ mime_type = request.get_json()['mimeType'] - (url, file_id) = user_files.prepare_for_drop(mime_type) + (url, file_id) = user_files.prepare_for_drop(mime_type, requires_cors=True) return { 'url': url, 'file_id': str(file_id), diff --git a/endpoints/api/discovery.py b/endpoints/api/discovery.py index db61a14b2..e0d8f5724 100644 --- a/endpoints/api/discovery.py +++ b/endpoints/api/discovery.py @@ -125,8 +125,17 @@ def swagger_route_data(include_internal=False, compact=False): if internal is not None: new_operation['internal'] = True + if include_internal: + requires_fresh_login = method_metadata(method, 'requires_fresh_login') + if requires_fresh_login is not None: + new_operation['requires_fresh_login'] = True + if not internal or (internal and include_internal): - operations.append(new_operation) + # Swagger requires valid nicknames on all operations. + if new_operation.get('nickname'): + operations.append(new_operation) + else: + logger.debug('Operation missing nickname: %s' % method) swagger_path = PARAM_REGEX.sub(r'{\2}', rule.rule) new_resource = { diff --git a/endpoints/api/image.py b/endpoints/api/image.py index 6da110eff..b2780c2aa 100644 --- a/endpoints/api/image.py +++ b/endpoints/api/image.py @@ -9,22 +9,33 @@ from data import model from util.cache import cache_control_flask_restful -def image_view(image): +def image_view(image, image_map): extended_props = image if image.storage and image.storage.id: extended_props = image.storage command = extended_props.command + + def docker_id(aid): + if not aid: + return '' + + return image_map[aid] + + # Calculate the ancestors string, with the DBID's replaced with the docker IDs. + ancestors = [docker_id(a) for a in image.ancestors.split('/')] + ancestors_string = '/'.join(ancestors) + return { 'id': image.docker_image_id, 'created': format_date(extended_props.created), 'comment': extended_props.comment, 'command': json.loads(command) if command else None, - 'ancestors': image.ancestors, - 'dbid': image.id, 'size': extended_props.image_size, 'locations': list(image.storage.locations), 'uploading': image.storage.uploading, + 'ancestors': ancestors_string, + 'sort_index': len(image.ancestors) } @@ -43,14 +54,16 @@ class RepositoryImageList(RepositoryParamResource): for tag in all_tags: tags_by_image_id[tag.image.docker_image_id].append(tag.name) + image_map = {} + for image in all_images: + image_map[str(image.id)] = image.docker_image_id def add_tags(image_json): image_json['tags'] = tags_by_image_id[image_json['id']] return image_json - return { - 'images': [add_tags(image_view(image)) for image in all_images] + 'images': [add_tags(image_view(image, image_map)) for image in all_images] } @@ -67,7 +80,12 @@ class RepositoryImage(RepositoryParamResource): if not image: raise NotFound() - return image_view(image) + # Lookup all the ancestor images for the image. + image_map = {} + for current_image in model.get_parent_images(namespace, repository, image): + image_map[str(current_image.id)] = image.docker_image_id + + return image_view(image, image_map) @resource('/v1/repository//image//changes') diff --git a/endpoints/api/repoemail.py b/endpoints/api/repoemail.py index 0b7c66917..288a4a12f 100644 --- a/endpoints/api/repoemail.py +++ b/endpoints/api/repoemail.py @@ -4,7 +4,7 @@ from flask import request, abort from endpoints.api import (resource, nickname, require_repo_admin, RepositoryParamResource, log_action, validate_json_request, NotFound, internal_only, - path_param) + path_param, show_if) from app import tf from data import model @@ -20,12 +20,13 @@ def record_view(record): return { 'email': record.email, 'repository': record.repository.name, - 'namespace': record.repository.namespace, + 'namespace': record.repository.namespace_user.username, 'confirmed': record.confirmed } @internal_only +@show_if(features.MAILING) @resource('/v1/repository//authorizedemail/') @path_param('repository', 'The full path of the repository. e.g. namespace/name') @path_param('email', 'The e-mail address') diff --git a/endpoints/api/repository.py b/endpoints/api/repository.py index b7ff899f6..722411d36 100644 --- a/endpoints/api/repository.py +++ b/endpoints/api/repository.py @@ -82,8 +82,7 @@ class RepositoryList(ApiResource): visibility = req['visibility'] - repo = model.create_repository(namespace_name, repository_name, owner, - visibility) + repo = model.create_repository(namespace_name, repository_name, owner, visibility) repo.description = req['description'] repo.save() @@ -112,7 +111,7 @@ class RepositoryList(ApiResource): """Fetch the list of repositories under a variety of situations.""" def repo_view(repo_obj): return { - 'namespace': repo_obj.namespace, + 'namespace': repo_obj.namespace_user.username, 'name': repo_obj.name, 'description': repo_obj.description, 'is_public': repo_obj.visibility.name == 'public', @@ -136,7 +135,8 @@ class RepositoryList(ApiResource): response['repositories'] = [repo_view(repo) for repo in repo_query if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, + repo.name).can())] return response @@ -171,8 +171,7 @@ class Repository(RepositoryParamResource): def tag_view(tag): return { 'name': tag.name, - 'image_id': tag.image.docker_image_id, - 'dbid': tag.image.id + 'image_id': tag.image.docker_image_id } organization = None diff --git a/endpoints/api/robot.py b/endpoints/api/robot.py index df01f1a0d..b52cd4c5b 100644 --- a/endpoints/api/robot.py +++ b/endpoints/api/robot.py @@ -35,6 +35,14 @@ class UserRobotList(ApiResource): @internal_only class UserRobot(ApiResource): """ Resource for managing a user's robots. """ + @require_user_admin + @nickname('getUserRobot') + def get(self, robot_shortname): + """ Returns the user's robot with the specified name. """ + parent = get_authenticated_user() + robot, password = model.get_robot(robot_shortname, parent) + return robot_view(robot.username, password) + @require_user_admin @nickname('createUserRobot') def put(self, robot_shortname): @@ -79,6 +87,18 @@ class OrgRobotList(ApiResource): @related_user_resource(UserRobot) class OrgRobot(ApiResource): """ Resource for managing an organization's robots. """ + @require_scope(scopes.ORG_ADMIN) + @nickname('getOrgRobot') + def get(self, orgname, robot_shortname): + """ Returns the organization's robot with the specified name. """ + permission = AdministerOrganizationPermission(orgname) + if permission.can(): + parent = model.get_organization(orgname) + robot, password = model.get_robot(robot_shortname, parent) + return robot_view(robot.username, password) + + raise Unauthorized() + @require_scope(scopes.ORG_ADMIN) @nickname('createOrgRobot') def put(self, orgname, robot_shortname): @@ -103,3 +123,38 @@ class OrgRobot(ApiResource): return 'Deleted', 204 raise Unauthorized() + + +@resource('/v1/user/robots//regenerate') +@path_param('robot_shortname', 'The short name for the robot, without any user or organization prefix') +@internal_only +class RegenerateUserRobot(ApiResource): + """ Resource for regenerate an organization's robot's token. """ + @require_user_admin + @nickname('regenerateUserRobotToken') + def post(self, robot_shortname): + """ Regenerates the token for a user's robot. """ + parent = get_authenticated_user() + robot, password = model.regenerate_robot_token(robot_shortname, parent) + log_action('regenerate_robot_token', parent.username, {'robot': robot_shortname}) + return robot_view(robot.username, password) + + +@resource('/v1/organization//robots//regenerate') +@path_param('orgname', 'The name of the organization') +@path_param('robot_shortname', 'The short name for the robot, without any user or organization prefix') +@related_user_resource(RegenerateUserRobot) +class RegenerateOrgRobot(ApiResource): + """ Resource for regenerate an organization's robot's token. """ + @require_scope(scopes.ORG_ADMIN) + @nickname('regenerateOrgRobotToken') + def post(self, orgname, robot_shortname): + """ Regenerates the token for an organization robot. """ + permission = AdministerOrganizationPermission(orgname) + if permission.can(): + parent = model.get_organization(orgname) + robot, password = model.regenerate_robot_token(robot_shortname, parent) + log_action('regenerate_robot_token', orgname, {'robot': robot_shortname}) + return robot_view(robot.username, password) + + raise Unauthorized() diff --git a/endpoints/api/search.py b/endpoints/api/search.py index 83a398b6b..3758f5f7c 100644 --- a/endpoints/api/search.py +++ b/endpoints/api/search.py @@ -112,7 +112,7 @@ class FindRepositories(ApiResource): def repo_view(repo): return { - 'namespace': repo.namespace, + 'namespace': repo.namespace_user.username, 'name': repo.name, 'description': repo.description } @@ -126,5 +126,5 @@ class FindRepositories(ApiResource): return { 'repositories': [repo_view(repo) for repo in matching if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace, repo.name).can())] + ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] } diff --git a/endpoints/api/subscribe.py b/endpoints/api/subscribe.py index dd6de9678..2c3fba359 100644 --- a/endpoints/api/subscribe.py +++ b/endpoints/api/subscribe.py @@ -15,6 +15,9 @@ logger = logging.getLogger(__name__) def carderror_response(exc): return {'carderror': exc.message}, 402 +def connection_response(exc): + return {'message': 'Could not contact Stripe. Please try again.'}, 503 + def subscription_view(stripe_subscription, used_repos): view = { @@ -74,19 +77,29 @@ def subscribe(user, plan, token, require_business_plan): log_action('account_change_plan', user.username, {'plan': plan}) except stripe.CardError as e: return carderror_response(e) + except stripe.APIConnectionError as e: + return connection_response(e) response_json = subscription_view(cus.subscription, private_repos) status_code = 201 else: # Change the plan - cus = billing.Customer.retrieve(user.stripe_id) + try: + cus = billing.Customer.retrieve(user.stripe_id) + except stripe.APIConnectionError as e: + return connection_response(e) if plan_found['price'] == 0: if cus.subscription is not None: # We only have to cancel the subscription if they actually have one - cus.cancel_subscription() - cus.save() + try: + cus.cancel_subscription() + cus.save() + except stripe.APIConnectionError as e: + return connection_response(e) + + check_repository_usage(user, plan_found) log_action('account_change_plan', user.username, {'plan': plan}) @@ -101,6 +114,8 @@ def subscribe(user, plan, token, require_business_plan): cus.save() except stripe.CardError as e: return carderror_response(e) + except stripe.APIConnectionError as e: + return connection_response(e) response_json = subscription_view(cus.subscription, private_repos) check_repository_usage(user, plan_found) diff --git a/endpoints/api/superuser.py b/endpoints/api/superuser.py index 3eacd2f97..6defe4c77 100644 --- a/endpoints/api/superuser.py +++ b/endpoints/api/superuser.py @@ -1,20 +1,22 @@ +import string import logging import json +from random import SystemRandom from app import app - from flask import request from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, log_action, internal_only, NotFound, require_user_admin, format_date, InvalidToken, require_scope, format_date, hide_if, show_if, parse_args, - query_param, abort, path_param) + query_param, abort, require_fresh_login, path_param) from endpoints.api.logs import get_logs from data import model from auth.permissions import SuperUserPermission from auth.auth_context import get_authenticated_user +from util.useremails import send_confirmation_email, send_recovery_email import features @@ -42,24 +44,6 @@ class SuperUserLogs(ApiResource): abort(403) -@resource('/v1/superuser/seats') -@internal_only -@show_if(features.SUPER_USERS) -@hide_if(features.BILLING) -class SeatUsage(ApiResource): - """ Resource for managing the seats granted in the license for the system. """ - @nickname('getSeatCount') - def get(self): - """ Returns the current number of seats being used in the system. """ - if SuperUserPermission().can(): - return { - 'count': model.get_active_user_count(), - 'allowed': app.config.get('LICENSE_USER_LIMIT', 0) - } - - abort(403) - - def user_view(user): return { 'username': user.username, @@ -73,6 +57,26 @@ def user_view(user): @show_if(features.SUPER_USERS) class SuperUserList(ApiResource): """ Resource for listing users in the system. """ + schemas = { + 'CreateInstallUser': { + 'id': 'CreateInstallUser', + 'description': 'Data for creating a user', + 'required': ['username', 'email'], + 'properties': { + 'username': { + 'type': 'string', + 'description': 'The username of the user being created' + }, + + 'email': { + 'type': 'string', + 'description': 'The email address of the user being created' + } + } + } + } + + @require_fresh_login @nickname('listAllUsers') def get(self): """ Returns a list of all users in the system. """ @@ -85,6 +89,63 @@ class SuperUserList(ApiResource): abort(403) + @require_fresh_login + @nickname('createInstallUser') + @validate_json_request('CreateInstallUser') + def post(self): + """ Creates a new user. """ + user_information = request.get_json() + if SuperUserPermission().can(): + username = user_information['username'] + email = user_information['email'] + + # Generate a temporary password for the user. + random = SystemRandom() + password = ''.join([random.choice(string.ascii_uppercase + string.digits) for _ in range(32)]) + + # Create the user. + user = model.create_user(username, password, email, auto_verify=not features.MAILING) + + # If mailing is turned on, send the user a verification email. + if features.MAILING: + confirmation = model.create_confirm_email_code(user, new_email=user.email) + send_confirmation_email(user.username, user.email, confirmation.code) + + return { + 'username': username, + 'email': email, + 'password': password + } + + abort(403) + + +@resource('/v1/superusers/users//sendrecovery') +@internal_only +@show_if(features.SUPER_USERS) +@show_if(features.MAILING) +class SuperUserSendRecoveryEmail(ApiResource): + """ Resource for sending a recovery user on behalf of a user. """ + @require_fresh_login + @nickname('sendInstallUserRecoveryEmail') + def post(self, username): + if SuperUserPermission().can(): + user = model.get_user(username) + if not user or user.organization or user.robot: + abort(404) + + if username in app.config['SUPER_USERS']: + abort(403) + + code = model.create_reset_password_email_code(user.email) + send_recovery_email(user.email, code.code) + return { + 'email': user.email + } + + abort(403) + + @resource('/v1/superuser/users/') @path_param('username', 'The username of the user being managed') @internal_only @@ -109,18 +170,20 @@ class SuperUserManagement(ApiResource): }, } + @require_fresh_login @nickname('getInstallUser') def get(self, username): """ Returns information about the specified user. """ if SuperUserPermission().can(): - user = model.get_user(username) - if not user or user.organization or user.robot: - abort(404) - - return user_view(user) + user = model.get_user(username) + if not user or user.organization or user.robot: + abort(404) + + return user_view(user) abort(403) + @require_fresh_login @nickname('deleteInstallUser') def delete(self, username): """ Deletes the specified user. """ @@ -137,6 +200,7 @@ class SuperUserManagement(ApiResource): abort(403) + @require_fresh_login @nickname('changeInstallUser') @validate_json_request('UpdateUser') def put(self, username): diff --git a/endpoints/api/tag.py b/endpoints/api/tag.py index 751fd6f60..223d57012 100644 --- a/endpoints/api/tag.py +++ b/endpoints/api/tag.py @@ -90,11 +90,14 @@ class RepositoryTagImages(RepositoryParamResource): raise NotFound() parent_images = model.get_parent_images(namespace, repository, tag_image) + image_map = {} + for image in parent_images: + image_map[str(image.id)] = image.docker_image_id parents = list(parent_images) parents.reverse() all_images = [tag_image] + parents return { - 'images': [image_view(image) for image in all_images] + 'images': [image_view(image, image_map) for image in all_images] } diff --git a/endpoints/api/team.py b/endpoints/api/team.py index 4ecef0e6e..e41ddad49 100644 --- a/endpoints/api/team.py +++ b/endpoints/api/team.py @@ -2,12 +2,51 @@ from flask import request from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error, log_action, Unauthorized, NotFound, internal_only, require_scope, - path_param) + path_param, query_param, truthy_bool, parse_args, require_user_admin, + show_if) from auth.permissions import AdministerOrganizationPermission, ViewTeamPermission from auth.auth_context import get_authenticated_user from auth import scopes from data import model +from util.useremails import send_org_invite_email +from util.gravatar import compute_hash +import features + +def try_accept_invite(code, user): + (team, inviter) = model.confirm_team_invite(code, user) + + model.delete_matching_notifications(user, 'org_team_invite', code=code) + + orgname = team.organization.username + log_action('org_team_member_invite_accepted', orgname, { + 'member': user.username, + 'team': team.name, + 'inviter': inviter.username + }) + + return team + + +def handle_addinvite_team(inviter, team, user=None, email=None): + invite = model.add_or_invite_to_team(inviter, team, user, email, + requires_invite = features.MAILING) + if not invite: + # User was added to the team directly. + return + + orgname = team.organization.username + if user: + model.create_notification('org_team_invite', user, metadata = { + 'code': invite.invite_token, + 'inviter': inviter.username, + 'org': orgname, + 'team': team.name + }) + + send_org_invite_email(user.username if user else email, user.email if user else email, + orgname, team.name, inviter.username, invite.invite_token) + return invite def team_view(orgname, team): view_permission = ViewTeamPermission(orgname, team.name) @@ -20,14 +59,28 @@ def team_view(orgname, team): 'role': role } -def member_view(member): +def member_view(member, invited=False): return { 'name': member.username, 'kind': 'user', 'is_robot': member.robot, + 'gravatar': compute_hash(member.email) if not member.robot else None, + 'invited': invited, } +def invite_view(invite): + if invite.user: + return member_view(invite.user, invited=True) + else: + return { + 'email': invite.email, + 'kind': 'invite', + 'gravatar': compute_hash(invite.email), + 'invited': True + } + + @resource('/v1/organization//team/') @path_param('orgname', 'The name of the organization') @path_param('teamname', 'The name of the team') @@ -119,10 +172,11 @@ class OrganizationTeam(ApiResource): @path_param('teamname', 'The name of the team') class TeamMemberList(ApiResource): """ Resource for managing the list of members for a team. """ - @require_scope(scopes.ORG_ADMIN) + @parse_args + @query_param('includePending', 'Whether to include pending members', type=truthy_bool, default=False) @nickname('getOrganizationTeamMembers') - def get(self, orgname, teamname): + def get(self, args, orgname, teamname): """ Retrieve the list of members for the specified team. """ view_permission = ViewTeamPermission(orgname, teamname) edit_permission = AdministerOrganizationPermission(orgname) @@ -135,11 +189,18 @@ class TeamMemberList(ApiResource): raise NotFound() members = model.get_organization_team_members(team.id) - return { - 'members': {m.username : member_view(m) for m in members}, + invites = [] + + if args['includePending'] and edit_permission.can(): + invites = model.get_organization_team_member_invites(team.id) + + data = { + 'members': [member_view(m) for m in members] + [invite_view(i) for i in invites], 'can_edit': edit_permission.can() } + return data + raise Unauthorized() @@ -153,7 +214,7 @@ class TeamMember(ApiResource): @require_scope(scopes.ORG_ADMIN) @nickname('updateOrganizationTeamMember') def put(self, orgname, teamname, membername): - """ Add a member to an existing team. """ + """ Adds or invites a member to an existing team. """ permission = AdministerOrganizationPermission(orgname) if permission.can(): team = None @@ -170,23 +231,151 @@ class TeamMember(ApiResource): if not user: raise request_error(message='Unknown user') - # Add the user to the team. - model.add_user_to_team(user, team) - log_action('org_add_team_member', orgname, {'member': membername, 'team': teamname}) - return member_view(user) + # Add or invite the user to the team. + inviter = get_authenticated_user() + invite = handle_addinvite_team(inviter, team, user=user) + if not invite: + log_action('org_add_team_member', orgname, {'member': membername, 'team': teamname}) + return member_view(user, invited=False) + + # User was invited. + log_action('org_invite_team_member', orgname, { + 'user': membername, + 'member': membername, + 'team': teamname + }) + return member_view(user, invited=True) raise Unauthorized() @require_scope(scopes.ORG_ADMIN) @nickname('deleteOrganizationTeamMember') def delete(self, orgname, teamname, membername): - """ Delete an existing member of a team. """ + """ Delete a member of a team. If the user is merely invited to join + the team, then the invite is removed instead. + """ permission = AdministerOrganizationPermission(orgname) if permission.can(): # Remote the user from the team. invoking_user = get_authenticated_user().username + + # Find the team. + try: + team = model.get_organization_team(orgname, teamname) + except model.InvalidTeamException: + raise NotFound() + + # Find the member. + member = model.get_user(membername) + if not member: + raise NotFound() + + # First attempt to delete an invite for the user to this team. If none found, + # then we try to remove the user directly. + if model.delete_team_user_invite(team, member): + log_action('org_delete_team_member_invite', orgname, { + 'user': membername, + 'team': teamname, + 'member': membername + }) + return 'Deleted', 204 + model.remove_user_from_team(orgname, teamname, membername, invoking_user) log_action('org_remove_team_member', orgname, {'member': membername, 'team': teamname}) return 'Deleted', 204 raise Unauthorized() + + +@resource('/v1/organization//team//invite/') +@show_if(features.MAILING) +class InviteTeamMember(ApiResource): + """ Resource for inviting a team member via email address. """ + @require_scope(scopes.ORG_ADMIN) + @nickname('inviteTeamMemberEmail') + def put(self, orgname, teamname, email): + """ Invites an email address to an existing team. """ + permission = AdministerOrganizationPermission(orgname) + if permission.can(): + team = None + + # Find the team. + try: + team = model.get_organization_team(orgname, teamname) + except model.InvalidTeamException: + raise NotFound() + + # Invite the email to the team. + inviter = get_authenticated_user() + invite = handle_addinvite_team(inviter, team, email=email) + log_action('org_invite_team_member', orgname, { + 'email': email, + 'team': teamname, + 'member': email + }) + return invite_view(invite) + + raise Unauthorized() + + @require_scope(scopes.ORG_ADMIN) + @nickname('deleteTeamMemberEmailInvite') + def delete(self, orgname, teamname, email): + """ Delete an invite of an email address to join a team. """ + permission = AdministerOrganizationPermission(orgname) + if permission.can(): + team = None + + # Find the team. + try: + team = model.get_organization_team(orgname, teamname) + except model.InvalidTeamException: + raise NotFound() + + # Delete the invite. + model.delete_team_email_invite(team, email) + log_action('org_delete_team_member_invite', orgname, { + 'email': email, + 'team': teamname, + 'member': email + }) + return 'Deleted', 204 + + raise Unauthorized() + + +@resource('/v1/teaminvite/') +@internal_only +@show_if(features.MAILING) +class TeamMemberInvite(ApiResource): + """ Resource for managing invites to jon a team. """ + @require_user_admin + @nickname('acceptOrganizationTeamInvite') + def put(self, code): + """ Accepts an invite to join a team in an organization. """ + # Accept the invite for the current user. + team = try_accept_invite(code, get_authenticated_user()) + if not team: + raise NotFound() + + orgname = team.organization.username + return { + 'org': orgname, + 'team': team.name + } + + @nickname('declineOrganizationTeamInvite') + @require_user_admin + def delete(self, code): + """ Delete an existing member of a team. """ + (team, inviter) = model.delete_team_invite(code, get_authenticated_user()) + + model.delete_matching_notifications(get_authenticated_user(), 'org_team_invite', code=code) + + orgname = team.organization.username + log_action('org_team_member_invite_declined', orgname, { + 'member': get_authenticated_user().username, + 'team': team.name, + 'inviter': inviter.username + }) + + return 'Deleted', 204 diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 4cc224a00..68cd18da8 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -15,7 +15,7 @@ from endpoints.api.build import (build_status_view, trigger_view, RepositoryBuil from endpoints.common import start_build from endpoints.trigger import (BuildTrigger as BuildTriggerBase, TriggerDeactivationException, TriggerActivationException, EmptyRepositoryException, - RepositoryReadException) + RepositoryReadException, TriggerStartException) from data import model from auth.permissions import UserAdminPermission, AdministerOrganizationPermission, ReadRepositoryPermission from util.names import parse_robot_username @@ -212,7 +212,7 @@ class BuildTriggerActivate(RepositoryParamResource): 'write') try: - repository_path = '%s/%s' % (trigger.repository.namespace, + repository_path = '%s/%s' % (trigger.repository.namespace_user.username, trigger.repository.name) path = url_for('webhooks.build_trigger_webhook', repository=repository_path, trigger_uuid=trigger.uuid) @@ -385,9 +385,24 @@ class BuildTriggerAnalyze(RepositoryParamResource): @path_param('trigger_uuid', 'The UUID of the build trigger') class ActivateBuildTrigger(RepositoryParamResource): """ Custom verb to manually activate a build trigger. """ + schemas = { + 'RunParameters': { + 'id': 'RunParameters', + 'type': 'object', + 'description': 'Optional run parameters for activating the build trigger', + 'additional_properties': False, + 'properties': { + 'branch_name': { + 'type': 'string', + 'description': '(GitHub Only) If specified, the name of the GitHub branch to build.' + } + } + } + } @require_repo_admin @nickname('manuallyStartBuildTrigger') + @validate_json_request('RunParameters') def post(self, namespace, repository, trigger_uuid): """ Manually start a build from the specified trigger. """ try: @@ -400,14 +415,18 @@ class ActivateBuildTrigger(RepositoryParamResource): if not handler.is_active(config_dict): raise InvalidRequest('Trigger is not active.') - specs = handler.manual_start(trigger.auth_token, config_dict) - dockerfile_id, tags, name, subdir = specs + try: + run_parameters = request.get_json() + specs = handler.manual_start(trigger.auth_token, config_dict, run_parameters=run_parameters) + dockerfile_id, tags, name, subdir = specs - repo = model.get_repository(namespace, repository) - pull_robot_name = model.get_pull_robot_name(trigger) + repo = model.get_repository(namespace, repository) + pull_robot_name = model.get_pull_robot_name(trigger) - build_request = start_build(repo, dockerfile_id, tags, name, subdir, True, - pull_robot_name=pull_robot_name) + build_request = start_build(repo, dockerfile_id, tags, name, subdir, True, + pull_robot_name=pull_robot_name) + except TriggerStartException as tse: + raise InvalidRequest(tse.message) resp = build_status_view(build_request, True) repo_string = '%s/%s' % (namespace, repository) @@ -437,6 +456,36 @@ class TriggerBuildList(RepositoryParamResource): } + +@resource('/v1/repository//trigger//fields/') +@internal_only +class BuildTriggerFieldValues(RepositoryParamResource): + """ Custom verb to fetch a values list for a particular field name. """ + @require_repo_admin + @nickname('listTriggerFieldValues') + def get(self, namespace, repository, trigger_uuid, field_name): + """ List the field values for a custom run field. """ + try: + trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + except model.InvalidBuildTriggerException: + raise NotFound() + + user_permission = UserAdminPermission(trigger.connected_user.username) + if user_permission.can(): + trigger_handler = BuildTriggerBase.get_trigger_for_service(trigger.service.name) + values = trigger_handler.list_field_values(trigger.auth_token, json.loads(trigger.config), + field_name) + + if values is None: + raise NotFound() + + return { + 'values': values + } + else: + raise Unauthorized() + + @resource('/v1/repository//trigger//sources') @path_param('repository', 'The full path of the repository. e.g. namespace/name') @path_param('trigger_uuid', 'The UUID of the build trigger') diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 3bafe40b4..440a76fa9 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -7,11 +7,13 @@ from flask.ext.principal import identity_changed, AnonymousIdentity from app import app, billing as stripe, authentication from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error, - log_action, internal_only, NotFound, require_user_admin, path_param, - InvalidToken, require_scope, format_date, hide_if, show_if, license_error, - define_json_response) + log_action, internal_only, NotFound, require_user_admin, parse_args, + query_param, InvalidToken, require_scope, format_date, hide_if, show_if, + license_error, require_fresh_login, path_param, define_json_response) from endpoints.api.subscribe import subscribe from endpoints.common import common_login +from endpoints.api.team import try_accept_invite + from data import model from data.billing import get_plan from auth.permissions import (AdministerOrganizationPermission, CreateRepositoryPermission, @@ -19,7 +21,8 @@ from auth.permissions import (AdministerOrganizationPermission, CreateRepository from auth.auth_context import get_authenticated_user from auth import scopes from util.gravatar import compute_hash -from util.useremails import (send_confirmation_email, send_recovery_email, send_change_email) +from util.useremails import (send_confirmation_email, send_recovery_email, send_change_email, send_password_changed) +from util.names import parse_single_urn import features @@ -40,9 +43,15 @@ def user_view(user): organizations = model.get_user_organizations(user.username) def login_view(login): + try: + metadata = json.loads(login.metadata_json) + except: + metadata = {} + return { 'service': login.service.name, 'service_identifier': login.service_ident, + 'metadata': metadata } logins = model.list_federated_logins(user) @@ -89,6 +98,7 @@ class User(ApiResource): """ Operations related to users. """ schemas = { 'NewUser': { + 'id': 'NewUser', 'type': 'object', 'description': 'Fields which must be specified for a new user.', @@ -185,6 +195,7 @@ class User(ApiResource): return user_view(user) @require_user_admin + @require_fresh_login @nickname('changeUserDetails') @internal_only @validate_json_request('UpdateUser') @@ -194,12 +205,15 @@ class User(ApiResource): user = get_authenticated_user() user_data = request.get_json() - try: + try: if 'password' in user_data: logger.debug('Changing password for user: %s', user.username) log_action('account_change_password', user.username) model.change_password(user, user_data['password']) + if features.MAILING: + send_password_changed(user.username, user.email) + if 'invoice_email' in user_data: logger.debug('Changing invoice_email for user: %s', user.username) model.change_invoice_email(user, user_data['invoice_email']) @@ -210,22 +224,30 @@ class User(ApiResource): # Email already used. raise request_error(message='E-mail address already used') - logger.debug('Sending email to change email address for user: %s', - user.username) - code = model.create_confirm_email_code(user, new_email=new_email) - send_change_email(user.username, user_data['email'], code.code) + if features.MAILING: + logger.debug('Sending email to change email address for user: %s', + user.username) + code = model.create_confirm_email_code(user, new_email=new_email) + send_change_email(user.username, user_data['email'], code.code) + else: + model.update_email(user, new_email, auto_verify=not features.MAILING) except model.InvalidPasswordException, ex: raise request_error(exception=ex) return user_view(user) + @show_if(features.USER_CREATION) @nickname('createNewUser') + @parse_args + @query_param('inviteCode', 'Invitation code given for creating the user.', type=str, + default='') @internal_only @validate_json_request('NewUser') - def post(self): + def post(self, args): """ Create a new user. """ user_data = request.get_json() + invite_code = args['inviteCode'] existing_user = model.get_user(user_data['username']) if existing_user: @@ -233,10 +255,29 @@ class User(ApiResource): try: new_user = model.create_user(user_data['username'], user_data['password'], - user_data['email']) - code = model.create_confirm_email_code(new_user) - send_confirmation_email(new_user.username, new_user.email, code.code) - return 'Created', 201 + user_data['email'], auto_verify=not features.MAILING) + + # Handle any invite codes. + parsed_invite = parse_single_urn(invite_code) + if parsed_invite is not None: + if parsed_invite[0] == 'teaminvite': + # Add the user to the team. + try: + try_accept_invite(invite_code, new_user) + except model.DataModelException: + pass + + + if features.MAILING: + code = model.create_confirm_email_code(new_user) + send_confirmation_email(new_user.username, new_user.email, code.code) + return { + 'awaiting_verification': True + } + else: + common_login(new_user) + return user_view(new_user) + except model.TooManyUsersException as ex: raise license_error(exception=ex) except model.DataModelException as ex: @@ -399,6 +440,37 @@ class Signin(ApiResource): return conduct_signin(username, password) +@resource('/v1/signin/verify') +@internal_only +class VerifyUser(ApiResource): + """ Operations for verifying the existing user. """ + schemas = { + 'VerifyUser': { + 'id': 'VerifyUser', + 'type': 'object', + 'description': 'Information required to verify the signed in user.', + 'required': [ + 'password', + ], + 'properties': { + 'password': { + 'type': 'string', + 'description': 'The user\'s password', + }, + }, + }, + } + + @require_user_admin + @nickname('verifyUser') + @validate_json_request('VerifyUser') + def post(self): + """ Verifies the signed in the user with the specified credentials. """ + signin_data = request.get_json() + password = signin_data['password'] + return conduct_signin(get_authenticated_user().username, password) + + @resource('/v1/signout') @internal_only class Signout(ApiResource): @@ -411,7 +483,21 @@ class Signout(ApiResource): return {'success': True} + +@resource('/v1/detachexternal/') +@internal_only +class DetachExternal(ApiResource): + """ Resource for detaching an external login. """ + @require_user_admin + @nickname('detachExternalLogin') + def post(self, servicename): + """ Request that the current user be detached from the external login service. """ + model.detach_external_login(get_authenticated_user(), servicename) + return {'success': True} + + @resource("/v1/recovery") +@show_if(features.MAILING) @internal_only class Recovery(ApiResource): """ Resource for requesting a password recovery email. """ @@ -446,11 +532,24 @@ class Recovery(ApiResource): @internal_only class UserNotificationList(ApiResource): @require_user_admin + @parse_args + @query_param('page', 'Offset page number. (int)', type=int, default=0) + @query_param('limit', 'Limit on the number of results (int)', type=int, default=5) @nickname('listUserNotifications') - def get(self): - notifications = model.list_notifications(get_authenticated_user()) + def get(self, args): + page = args['page'] + limit = args['limit'] + + notifications = list(model.list_notifications(get_authenticated_user(), page=page, limit=limit + 1)) + has_more = False + + if len(notifications) > limit: + has_more = True + notifications = notifications[0:limit] + return { - 'notifications': [notification_view(notification) for notification in notifications] + 'notifications': [notification_view(notification) for notification in notifications], + 'additional': has_more } diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index 015f3c3a7..637033ab6 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -4,12 +4,14 @@ from flask import request, redirect, url_for, Blueprint from flask.ext.login import current_user from endpoints.common import render_page_template, common_login, route_show_if -from app import app, analytics +from app import app, analytics, get_app_url from data import model from util.names import parse_repository_name +from util.validation import generate_valid_usernames from util.http import abort from auth.permissions import AdministerRepositoryPermission from auth.auth import require_session_login +from peewee import IntegrityError import features @@ -20,20 +22,40 @@ client = app.config['HTTPCLIENT'] callback = Blueprint('callback', __name__) +def render_ologin_error(service_name, + error_message='Could not load user data. The token may have expired.'): + return render_page_template('ologinerror.html', service_name=service_name, + error_message=error_message, + service_url=get_app_url(), + user_creation=features.USER_CREATION) -def exchange_github_code_for_token(code, for_login=True): +def exchange_code_for_token(code, service_name='GITHUB', for_login=True, form_encode=False, + redirect_suffix=''): code = request.args.get('code') + id_config = service_name + '_LOGIN_CLIENT_ID' if for_login else service_name + '_CLIENT_ID' + secret_config = service_name + '_LOGIN_CLIENT_SECRET' if for_login else service_name + '_CLIENT_SECRET' + payload = { - 'client_id': app.config['GITHUB_LOGIN_CLIENT_ID' if for_login else 'GITHUB_CLIENT_ID'], - 'client_secret': app.config['GITHUB_LOGIN_CLIENT_SECRET' if for_login else 'GITHUB_CLIENT_SECRET'], + 'client_id': app.config[id_config], + 'client_secret': app.config[secret_config], 'code': code, + 'grant_type': 'authorization_code', + 'redirect_uri': '%s://%s/oauth2/%s/callback%s' % (app.config['PREFERRED_URL_SCHEME'], + app.config['SERVER_HOSTNAME'], + service_name.lower(), + redirect_suffix) } + headers = { 'Accept': 'application/json' } - get_access_token = client.post(app.config['GITHUB_TOKEN_URL'], - params=payload, headers=headers) + if form_encode: + get_access_token = client.post(app.config[service_name + '_TOKEN_URL'], + data=payload, headers=headers) + else: + get_access_token = client.post(app.config[service_name + '_TOKEN_URL'], + params=payload, headers=headers) json_data = get_access_token.json() if not json_data: @@ -52,17 +74,87 @@ def get_github_user(token): return get_user.json() +def get_google_user(token): + token_param = { + 'access_token': token, + 'alt': 'json', + } + + get_user = client.get(app.config['GOOGLE_USER_URL'], params=token_param) + return get_user.json() + +def conduct_oauth_login(service_name, user_id, username, email, metadata={}): + to_login = model.verify_federated_login(service_name.lower(), user_id) + if not to_login: + # See if we can create a new user. + if not features.USER_CREATION: + error_message = 'User creation is disabled. Please contact your administrator' + return render_ologin_error(service_name, error_message) + + # Try to create the user + try: + valid = next(generate_valid_usernames(username)) + to_login = model.create_federated_user(valid, email, service_name.lower(), + user_id, set_password_notification=True, + metadata=metadata) + + # Success, tell analytics + analytics.track(to_login.username, 'register', {'service': service_name.lower()}) + + state = request.args.get('state', None) + if state: + logger.debug('Aliasing with state: %s' % state) + analytics.alias(to_login.username, state) + + except model.DataModelException, ex: + return render_ologin_error(service_name, ex.message) + + if common_login(to_login): + return redirect(url_for('web.index')) + + return render_ologin_error(service_name) + +def get_google_username(user_data): + username = user_data['email'] + at = username.find('@') + if at > 0: + username = username[0:at] + + return username + + +@callback.route('/google/callback', methods=['GET']) +@route_show_if(features.GOOGLE_LOGIN) +def google_oauth_callback(): + error = request.args.get('error', None) + if error: + return render_ologin_error('Google', error) + + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', form_encode=True) + user_data = get_google_user(token) + if not user_data or not user_data.get('id', None) or not user_data.get('email', None): + return render_ologin_error('Google') + + username = get_google_username(user_data) + metadata = { + 'service_username': user_data['email'] + } + + return conduct_oauth_login('Google', user_data['id'], username, user_data['email'], + metadata=metadata) + + @callback.route('/github/callback', methods=['GET']) @route_show_if(features.GITHUB_LOGIN) def github_oauth_callback(): error = request.args.get('error', None) if error: - return render_page_template('githuberror.html', error_message=error) + return render_ologin_error('GitHub', error) - token = exchange_github_code_for_token(request.args.get('code')) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) - if not user_data: - return render_page_template('githuberror.html', error_message='Could not load user data') + if not user_data or not 'login' in user_data: + return render_ologin_error('GitHub') username = user_data['login'] github_id = user_data['id'] @@ -84,42 +176,67 @@ def github_oauth_callback(): if user_email['primary']: break - to_login = model.verify_federated_login('github', github_id) - if not to_login: - # try to create the user - try: - to_login = model.create_federated_user(username, found_email, 'github', - github_id, set_password_notification=True) + metadata = { + 'service_username': username + } - # Success, tell analytics - analytics.track(to_login.username, 'register', {'service': 'github'}) + return conduct_oauth_login('github', github_id, username, found_email, metadata=metadata) - state = request.args.get('state', None) - if state: - logger.debug('Aliasing with state: %s' % state) - analytics.alias(to_login.username, state) - except model.DataModelException, ex: - return render_page_template('githuberror.html', error_message=ex.message) +@callback.route('/google/callback/attach', methods=['GET']) +@route_show_if(features.GOOGLE_LOGIN) +@require_session_login +def google_oauth_attach(): + token = exchange_code_for_token(request.args.get('code'), service_name='GOOGLE', + redirect_suffix='/attach', form_encode=True) - if common_login(to_login): - return redirect(url_for('web.index')) + user_data = get_google_user(token) + if not user_data or not user_data.get('id', None): + return render_ologin_error('Google') - return render_page_template('githuberror.html') + google_id = user_data['id'] + user_obj = current_user.db_user() + + username = get_google_username(user_data) + metadata = { + 'service_username': user_data['email'] + } + + try: + model.attach_federated_login(user_obj, 'google', google_id, metadata=metadata) + except IntegrityError: + err = 'Google account %s is already attached to a %s account' % ( + username, app.config['REGISTRY_TITLE_SHORT']) + return render_ologin_error('Google', err) + + return redirect(url_for('web.user')) @callback.route('/github/callback/attach', methods=['GET']) @route_show_if(features.GITHUB_LOGIN) @require_session_login def github_oauth_attach(): - token = exchange_github_code_for_token(request.args.get('code')) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB') user_data = get_github_user(token) if not user_data: - return render_page_template('githuberror.html', error_message='Could not load user data') + return render_ologin_error('GitHub') github_id = user_data['id'] user_obj = current_user.db_user() - model.attach_federated_login(user_obj, 'github', github_id) + + username = user_data['login'] + metadata = { + 'service_username': username + } + + try: + model.attach_federated_login(user_obj, 'github', github_id, metadata=metadata) + except IntegrityError: + err = 'Github account %s is already attached to a %s account' % ( + username, app.config['REGISTRY_TITLE_SHORT']) + + return render_ologin_error('GitHub', err) + return redirect(url_for('web.user')) @@ -130,7 +247,8 @@ def github_oauth_attach(): def attach_github_build_trigger(namespace, repository): permission = AdministerRepositoryPermission(namespace, repository) if permission.can(): - token = exchange_github_code_for_token(request.args.get('code'), for_login=False) + token = exchange_code_for_token(request.args.get('code'), service_name='GITHUB', + for_login=False) repo = model.get_repository(namespace, repository) if not repo: msg = 'Invalid repository: %s/%s' % (namespace, repository) diff --git a/endpoints/common.py b/endpoints/common.py index fe09104ca..37ae80ee8 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -2,8 +2,9 @@ import logging import urlparse import json import string +import datetime -from flask import make_response, render_template, request, abort +from flask import make_response, render_template, request, abort, session from flask.ext.login import login_user, UserMixin from flask.ext.principal import identity_changed from random import SystemRandom @@ -81,20 +82,23 @@ def param_required(param_name): @login_manager.user_loader -def load_user(username): - logger.debug('User loader loading deferred user: %s' % username) - return _LoginWrappedDBUser(username) +def load_user(user_db_id): + logger.debug('User loader loading deferred user id: %s' % user_db_id) + try: + user_db_id_int = int(user_db_id) + return _LoginWrappedDBUser(user_db_id_int) + except ValueError: + return None class _LoginWrappedDBUser(UserMixin): - def __init__(self, db_username, db_user=None): - - self._db_username = db_username + def __init__(self, user_db_id, db_user=None): + self._db_id = user_db_id self._db_user = db_user def db_user(self): if not self._db_user: - self._db_user = model.get_user(self._db_username) + self._db_user = model.get_user_by_id(self._db_id) return self._db_user def is_authenticated(self): @@ -104,14 +108,15 @@ class _LoginWrappedDBUser(UserMixin): return self.db_user().verified def get_id(self): - return unicode(self._db_username) + return unicode(self._db_id) def common_login(db_user): - if login_user(_LoginWrappedDBUser(db_user.username, db_user)): + if login_user(_LoginWrappedDBUser(db_user.id, db_user)): logger.debug('Successfully signed in as: %s' % db_user.username) - new_identity = QuayDeferredPermissionUser(db_user.username, 'username', {scopes.DIRECT_LOGIN}) + new_identity = QuayDeferredPermissionUser(db_user.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) + session['login_time'] = datetime.datetime.now() return True else: logger.debug('User could not be logged in, inactive?.') @@ -200,7 +205,7 @@ def check_repository_usage(user_or_org, plan_found): def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, trigger=None, pull_robot_name=None): host = urlparse.urlparse(request.url).netloc - repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name) + repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name) token = model.create_access_token(repository, 'write') logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', @@ -216,9 +221,9 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, dockerfile_id, build_name, trigger, pull_robot_name=pull_robot_name) - dockerfile_build_queue.put([repository.namespace, repository.name], json.dumps({ + dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json.dumps({ 'build_uuid': build_request.uuid, - 'namespace': repository.namespace, + 'namespace': repository.namespace_user.username, 'repository': repository.name, 'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None }), retries_remaining=1) @@ -226,7 +231,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, # Add the build to the repo's log. metadata = { 'repo': repository.name, - 'namespace': repository.namespace, + 'namespace': repository.namespace_user.username, 'fileid': dockerfile_id, 'manual': manual, } @@ -236,9 +241,8 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, metadata['config'] = json.loads(trigger.config) metadata['service'] = trigger.service.name - model.log_action('build_dockerfile', repository.namespace, - ip=request.remote_addr, metadata=metadata, - repository=repository) + model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr, + metadata=metadata, repository=repository) # Add notifications for the build queue. profile.debug('Adding notifications for repository') diff --git a/endpoints/index.py b/endpoints/index.py index 39327e6a8..5f0a88695 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -19,6 +19,7 @@ from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission, from util.http import abort from endpoints.notificationhelper import spawn_notification +import features logger = logging.getLogger(__name__) profile = logging.getLogger('application.profiler') @@ -65,7 +66,13 @@ def generate_headers(role='read'): @index.route('/users', methods=['POST']) @index.route('/users/', methods=['POST']) def create_user(): + if not features.USER_CREATION: + abort(400, 'User creation is disabled. Please speak to your administrator.') + user_data = request.get_json() + if not 'username' in user_data: + abort(400, 'Missing username') + username = user_data['username'] password = user_data.get('password', '') @@ -413,8 +420,39 @@ def put_repository_auth(namespace, repository): @index.route('/search', methods=['GET']) +@process_auth def get_search(): - abort(501, 'Not Implemented', issue='not-implemented') + def result_view(repo): + return { + "name": repo.namespace_user.username + '/' + repo.name, + "description": repo.description + } + + query = request.args.get('q') + + username = None + user = get_authenticated_user() + if user is not None: + username = user.username + + if query: + matching = model.get_matching_repositories(query, username) + else: + matching = [] + + results = [result_view(repo) for repo in matching + if (repo.visibility.name == 'public' or + ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] + + data = { + "query": query, + "num_results": len(results), + "results" : results + } + + resp = make_response(json.dumps(data), 200) + resp.mimetype = 'application/json' + return resp @index.route('/_ping') diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index f1cbec42c..4a195fbd7 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -1,8 +1,4 @@ import logging -import io -import os.path -import tarfile -import base64 from notificationhelper import build_event_data @@ -15,6 +11,13 @@ class NotificationEvent(object): def __init__(self): pass + def get_level(self, event_data, notification_data): + """ + Returns a 'level' representing the severity of the event. + Valid values are: 'info', 'warning', 'error', 'primary' + """ + raise NotImplementedError + def get_summary(self, event_data, notification_data): """ Returns a human readable one-line summary for the given notification data. @@ -55,6 +58,9 @@ class RepoPushEvent(NotificationEvent): def event_name(cls): return 'repo_push' + def get_level(self, event_data, notification_data): + return 'info' + def get_summary(self, event_data, notification_data): return 'Repository %s updated' % (event_data['repository']) @@ -87,6 +93,9 @@ class BuildQueueEvent(NotificationEvent): @classmethod def event_name(cls): return 'build_queued' + + def get_level(self, event_data, notification_data): + return 'info' def get_sample_data(self, repository): build_uuid = 'fake-build-id' @@ -127,6 +136,9 @@ class BuildStartEvent(NotificationEvent): def event_name(cls): return 'build_start' + def get_level(self, event_data, notification_data): + return 'info' + def get_sample_data(self, repository): build_uuid = 'fake-build-id' @@ -155,6 +167,9 @@ class BuildSuccessEvent(NotificationEvent): def event_name(cls): return 'build_success' + def get_level(self, event_data, notification_data): + return 'primary' + def get_sample_data(self, repository): build_uuid = 'fake-build-id' @@ -183,7 +198,12 @@ class BuildFailureEvent(NotificationEvent): def event_name(cls): return 'build_failure' + def get_level(self, event_data, notification_data): + return 'error' + def get_sample_data(self, repository): + build_uuid = 'fake-build-id' + return build_event_data(repository, { 'build_id': build_uuid, 'build_name': 'some-fake-build', diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py index 773779fb7..6f80f83d0 100644 --- a/endpoints/notificationhelper.py +++ b/endpoints/notificationhelper.py @@ -4,7 +4,7 @@ from data import model import json def build_event_data(repo, extra_data={}, subpage=None): - repo_string = '%s/%s' % (repo.namespace, repo.name) + repo_string = '%s/%s' % (repo.namespace_user.username, repo.name) homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], repo_string) @@ -17,7 +17,7 @@ def build_event_data(repo, extra_data={}, subpage=None): event_data = { 'repository': repo_string, - 'namespace': repo.namespace, + 'namespace': repo.namespace_user.username, 'name': repo.name, 'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string), 'homepage': homepage, @@ -30,7 +30,7 @@ def build_event_data(repo, extra_data={}, subpage=None): def build_notification_data(notification, event_data): return { 'notification_uuid': notification.uuid, - 'repository_namespace': notification.repository.namespace, + 'repository_namespace': notification.repository.namespace_user.username, 'repository_name': notification.repository.name, 'event_data': event_data } @@ -39,8 +39,9 @@ def build_notification_data(notification, event_data): def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]): event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage) - notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name) + notifications = model.list_repo_notifications(repo.namespace_user.username, repo.name, + event_name=event_name) for notification in notifications: notification_data = build_notification_data(notification, event_data) - path = [repo.namespace, repo.name, event_name] + pathargs + path = [repo.namespace_user.username, repo.name, event_name] + pathargs notification_queue.put(path, json.dumps(notification_data)) diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index b49055157..589ebd06d 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -4,10 +4,13 @@ import os.path import tarfile import base64 import json +import requests +import re from flask.ext.mail import Message -from app import mail, app +from app import mail, app, get_app_url from data import model +from workers.worker import JobException logger = logging.getLogger(__name__) @@ -17,6 +20,9 @@ class InvalidNotificationMethodException(Exception): class CannotValidateNotificationMethodException(Exception): pass +class NotificationMethodPerformException(JobException): + pass + class NotificationMethod(object): def __init__(self): @@ -82,7 +88,7 @@ class QuayNotificationMethod(NotificationMethod): return (True, 'Unknown organization %s' % target_info['name'], None) # Only repositories under the organization can cause notifications to that org. - if target_info['name'] != repository.namespace: + if target_info['name'] != repository.namespace_user.username: return (False, 'Organization name must match repository namespace') return (True, None, [target]) @@ -90,7 +96,7 @@ class QuayNotificationMethod(NotificationMethod): # Lookup the team. team = None try: - team = model.get_organization_team(repository.namespace, target_info['name']) + team = model.get_organization_team(repository.namespace_user.username, target_info['name']) except model.InvalidTeamException: # Probably deleted. return (True, 'Unknown team %s' % target_info['name'], None) @@ -103,19 +109,18 @@ class QuayNotificationMethod(NotificationMethod): repository = notification.repository if not repository: # Probably deleted. - return True + return # Lookup the target user or team to which we'll send the notification. config_data = json.loads(notification.config_json) status, err_message, target_users = self.find_targets(repository, config_data) if not status: - return False + raise NotificationMethodPerformException(err_message) # For each of the target users, create a notification. for target_user in set(target_users or []): model.create_notification(event_handler.event_name(), target_user, metadata=notification_data['event_data']) - return True class EmailMethod(NotificationMethod): @@ -128,7 +133,8 @@ class EmailMethod(NotificationMethod): if not email: raise CannotValidateNotificationMethodException('Missing e-mail address') - record = model.get_email_authorized_for_repo(repository.namespace, repository.name, email) + record = model.get_email_authorized_for_repo(repository.namespace_user.username, + repository.name, email) if not record or not record.confirmed: raise CannotValidateNotificationMethodException('The specified e-mail address ' 'is not authorized to receive ' @@ -139,7 +145,7 @@ class EmailMethod(NotificationMethod): config_data = json.loads(notification.config_json) email = config_data.get('email', '') if not email: - return False + return msg = Message(event_handler.get_summary(notification_data['event_data'], notification_data), sender='support@quay.io', @@ -151,9 +157,7 @@ class EmailMethod(NotificationMethod): mail.send(msg) except Exception as ex: logger.exception('Email was unable to be sent: %s' % ex.message) - return False - - return True + raise NotificationMethodPerformException(ex.message) class WebhookMethod(NotificationMethod): @@ -170,7 +174,7 @@ class WebhookMethod(NotificationMethod): config_data = json.loads(notification.config_json) url = config_data.get('url', '') if not url: - return False + return payload = notification_data['event_data'] headers = {'Content-type': 'application/json'} @@ -178,12 +182,197 @@ class WebhookMethod(NotificationMethod): try: resp = requests.post(url, data=json.dumps(payload), headers=headers) if resp.status_code/100 != 2: - logger.error('%s response for webhook to url: %s' % (resp.status_code, - url)) - return False + error_message = '%s response for webhook to url: %s' % (resp.status_code, url) + logger.error(error_message) + logger.error(resp.content) + raise NotificationMethodPerformException(error_message) except requests.exceptions.RequestException as ex: logger.exception('Webhook was unable to be sent: %s' % ex.message) - return False + raise NotificationMethodPerformException(ex.message) - return True + +class FlowdockMethod(NotificationMethod): + """ Method for sending notifications to Flowdock via the Team Inbox API: + https://www.flowdock.com/api/team-inbox + """ + @classmethod + def method_name(cls): + return 'flowdock' + + def validate(self, repository, config_data): + token = config_data.get('flow_api_token', '') + if not token: + raise CannotValidateNotificationMethodException('Missing Flowdock API Token') + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + token = config_data.get('flow_api_token', '') + if not token: + return + + owner = model.get_user(notification.repository.namespace_user.username) + if not owner: + # Something went wrong. + return + + url = 'https://api.flowdock.com/v1/messages/team_inbox/%s' % token + headers = {'Content-type': 'application/json'} + payload = { + 'source': 'Quay', + 'from_address': 'support@quay.io', + 'subject': event_handler.get_summary(notification_data['event_data'], notification_data), + 'content': event_handler.get_message(notification_data['event_data'], notification_data), + 'from_name': owner.username, + 'project': (notification.repository.namespace_user.username + ' ' + + notification.repository.name), + 'tags': ['#' + event_handler.event_name()], + 'link': notification_data['event_data']['homepage'] + } + + try: + resp = requests.post(url, data=json.dumps(payload), headers=headers) + if resp.status_code/100 != 2: + error_message = '%s response for flowdock to url: %s' % (resp.status_code, url) + logger.error(error_message) + logger.error(resp.content) + raise NotificationMethodPerformException(error_message) + + except requests.exceptions.RequestException as ex: + logger.exception('Flowdock method was unable to be sent: %s' % ex.message) + raise NotificationMethodPerformException(ex.message) + + +class HipchatMethod(NotificationMethod): + """ Method for sending notifications to Hipchat via the API: + https://www.hipchat.com/docs/apiv2/method/send_room_notification + """ + @classmethod + def method_name(cls): + return 'hipchat' + + def validate(self, repository, config_data): + if not config_data.get('notification_token', ''): + raise CannotValidateNotificationMethodException('Missing Hipchat Room Notification Token') + + if not config_data.get('room_id', ''): + raise CannotValidateNotificationMethodException('Missing Hipchat Room ID') + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + + token = config_data.get('notification_token', '') + room_id = config_data.get('room_id', '') + + if not token or not room_id: + return + + owner = model.get_user(notification.repository.namespace_user.username) + if not owner: + # Something went wrong. + return + + url = 'https://api.hipchat.com/v2/room/%s/notification?auth_token=%s' % (room_id, token) + + level = event_handler.get_level(notification_data['event_data'], notification_data) + color = { + 'info': 'gray', + 'warning': 'yellow', + 'error': 'red', + 'primary': 'purple' + }.get(level, 'gray') + + headers = {'Content-type': 'application/json'} + payload = { + 'color': color, + 'message': event_handler.get_message(notification_data['event_data'], notification_data), + 'notify': level == 'error', + 'message_format': 'html', + } + + try: + resp = requests.post(url, data=json.dumps(payload), headers=headers) + if resp.status_code/100 != 2: + error_message = '%s response for hipchat to url: %s' % (resp.status_code, url) + logger.error(error_message) + logger.error(resp.content) + raise NotificationMethodPerformException(error_message) + + except requests.exceptions.RequestException as ex: + logger.exception('Hipchat method was unable to be sent: %s' % ex.message) + raise NotificationMethodPerformException(ex.message) + + +class SlackMethod(NotificationMethod): + """ Method for sending notifications to Slack via the API: + https://api.slack.com/docs/attachments + """ + @classmethod + def method_name(cls): + return 'slack' + + def validate(self, repository, config_data): + if not config_data.get('token', ''): + raise CannotValidateNotificationMethodException('Missing Slack Token') + + if not config_data.get('subdomain', '').isalnum(): + raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name') + + def formatForSlack(self, message): + message = message.replace('\n', '') + message = re.sub(r'\s+', ' ', message) + message = message.replace('
', '\n') + message = re.sub(r'(.+)', '<\\1|\\2>', message) + return message + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + + token = config_data.get('token', '') + subdomain = config_data.get('subdomain', '') + + if not token or not subdomain: + return + + owner = model.get_user(notification.repository.namespace_user.username) + if not owner: + # Something went wrong. + return + + url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token) + + level = event_handler.get_level(notification_data['event_data'], notification_data) + color = { + 'info': '#ffffff', + 'warning': 'warning', + 'error': 'danger', + 'primary': 'good' + }.get(level, '#ffffff') + + summary = event_handler.get_summary(notification_data['event_data'], notification_data) + message = event_handler.get_message(notification_data['event_data'], notification_data) + + headers = {'Content-type': 'application/json'} + payload = { + 'text': summary, + 'username': 'quayiobot', + 'attachments': [ + { + 'fallback': summary, + 'text': self.formatForSlack(message), + 'color': color + } + ] + } + + try: + resp = requests.post(url, data=json.dumps(payload), headers=headers) + if resp.status_code/100 != 2: + error_message = '%s response for Slack to url: %s' % (resp.status_code, url) + logger.error(error_message) + logger.error(resp.content) + raise NotificationMethodPerformException(error_message) + + except requests.exceptions.RequestException as ex: + logger.exception('Slack method was unable to be sent: %s' % ex.message) + raise NotificationMethodPerformException(ex.message) diff --git a/endpoints/registry.py b/endpoints/registry.py index 72633939e..5699f0db2 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -14,6 +14,7 @@ from util.http import abort, exact_abort from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission) from data import model +from util import gzipstream registry = Blueprint('registry', __name__) @@ -110,10 +111,10 @@ def head_image_layer(namespace, repository, image_id, headers): extra_headers = {} - # Add the Accept-Ranges header if the storage engine supports resumeable + # Add the Accept-Ranges header if the storage engine supports resumable # downloads. - if store.get_supports_resumeable_downloads(repo_image.storage.locations): - profile.debug('Storage supports resumeable downloads') + if store.get_supports_resumable_downloads(repo_image.storage.locations): + profile.debug('Storage supports resumable downloads') extra_headers['Accept-Ranges'] = 'bytes' resp = make_response('') @@ -193,21 +194,33 @@ def put_image_layer(namespace, repository, image_id): # encoding (Gunicorn) input_stream = request.environ['wsgi.input'] - # compute checksums - csums = [] + # Create a socket reader to read the input stream containing the layer data. sr = SocketReader(input_stream) + + # Add a handler that store the data in storage. tmp, store_hndlr = store.temp_store_handler() sr.add_handler(store_hndlr) + + # Add a handler to compute the uncompressed size of the layer. + uncompressed_size_info, size_hndlr = gzipstream.calculate_size_handler() + sr.add_handler(size_hndlr) + + # Add a handler which computes the checksum. h, sum_hndlr = checksums.simple_checksum_handler(json_data) sr.add_handler(sum_hndlr) + + # Stream write the data to storage. store.stream_write(repo_image.storage.locations, layer_path, sr) + + # Append the computed checksum. + csums = [] csums.append('sha256:{0}'.format(h.hexdigest())) try: image_size = tmp.tell() # Save the size of the image. - model.set_image_size(image_id, namespace, repository, image_size) + model.set_image_size(image_id, namespace, repository, image_size, uncompressed_size_info.size) tmp.seek(0) csums.append(checksums.compute_tarsum(tmp, json_data)) @@ -451,11 +464,6 @@ def put_image_json(namespace, repository, image_id): set_uploading_flag(repo_image, True) - # We cleanup any old checksum in case it's a retry after a fail - profile.debug('Cleanup old checksum') - repo_image.storage.checksum = None - repo_image.storage.save() - # If we reach that point, it means that this is a new image or a retry # on a failed push # save the metadata diff --git a/endpoints/trigger.py b/endpoints/trigger.py index ab7aa9065..4a10485ae 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -36,6 +36,9 @@ class TriggerActivationException(Exception): class TriggerDeactivationException(Exception): pass +class TriggerStartException(Exception): + pass + class ValidationRequestException(Exception): pass @@ -109,12 +112,19 @@ class BuildTrigger(object): """ raise NotImplementedError - def manual_start(self, auth_token, config): + def manual_start(self, auth_token, config, run_parameters = None): """ Manually creates a repository build for this trigger. """ raise NotImplementedError + def list_field_values(self, auth_token, config, field_name): + """ + Lists all values for the given custom trigger field. For example, a trigger might have a + field named "branches", and this method would return all branches. + """ + raise NotImplementedError + @classmethod def service_name(cls): """ @@ -291,6 +301,9 @@ class GithubBuildTrigger(BuildTrigger): with tarfile.open(fileobj=tarball) as archive: tarball_subdir = archive.getnames()[0] + # Seek to position 0 to make boto multipart happy + tarball.seek(0) + dockerfile_id = user_files.store_file(tarball, TARBALL_MIME) logger.debug('Successfully prepared job') @@ -342,14 +355,37 @@ class GithubBuildTrigger(BuildTrigger): return GithubBuildTrigger._prepare_build(config, repo, commit_sha, short_sha, ref) - def manual_start(self, auth_token, config): - source = config['build_source'] + def manual_start(self, auth_token, config, run_parameters = None): + try: + source = config['build_source'] + run_parameters = run_parameters or {} - gh_client = self._get_client(auth_token) - repo = gh_client.get_repo(source) - master = repo.get_branch(repo.default_branch) - master_sha = master.commit.sha - short_sha = GithubBuildTrigger.get_display_name(master_sha) - ref = 'refs/heads/%s' % repo.default_branch + gh_client = self._get_client(auth_token) + repo = gh_client.get_repo(source) + master = repo.get_branch(repo.default_branch) + master_sha = master.commit.sha + short_sha = GithubBuildTrigger.get_display_name(master_sha) + ref = 'refs/heads/%s' % (run_parameters.get('branch_name') or repo.default_branch) - return self._prepare_build(config, repo, master_sha, short_sha, ref) + return self._prepare_build(config, repo, master_sha, short_sha, ref) + except GithubException as ghe: + raise TriggerStartException(ghe.data['message']) + + + def list_field_values(self, auth_token, config, field_name): + if field_name == 'branch_name': + gh_client = self._get_client(auth_token) + source = config['build_source'] + repo = gh_client.get_repo(source) + branches = [branch['name'] for branch in repo.get_branches()] + + if not repo.default_branch in branches: + branches.insert(0, repo.default_branch) + + if branches[0] != repo.default_branch: + branches.remove(repo.default_branch) + branches.insert(0, repo.default_branch) + + return branches + + return None diff --git a/endpoints/web.py b/endpoints/web.py index 19f9bb7f1..63e463666 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -18,6 +18,7 @@ from endpoints.common import common_login, render_page_template, route_show_if, from endpoints.csrf import csrf_protect, generate_csrf_token from util.names import parse_repository_name from util.gravatar import compute_hash +from util.useremails import send_email_changed from auth import scopes import features @@ -32,8 +33,8 @@ STATUS_TAGS = app.config['STATUS_TAGS'] @web.route('/', methods=['GET'], defaults={'path': ''}) @web.route('/organization/', methods=['GET']) @no_cache -def index(path): - return render_page_template('index.html') +def index(path, **kwargs): + return render_page_template('index.html', **kwargs) @web.route('/500', methods=['GET']) @@ -101,7 +102,7 @@ def superuser(): @web.route('/signin/') @no_cache -def signin(): +def signin(redirect=None): return index('') @@ -123,6 +124,13 @@ def new(): return index('') +@web.route('/confirminvite') +@no_cache +def confirm_invite(): + code = request.values['code'] + return index('', code=code) + + @web.route('/repository/', defaults={'path': ''}) @web.route('/repository/', methods=['GET']) @no_cache @@ -215,6 +223,7 @@ def receipt(): @web.route('/authrepoemail', methods=['GET']) +@route_show_if(features.MAILING) def confirm_repo_email(): code = request.values['code'] record = None @@ -228,23 +237,27 @@ def confirm_repo_email(): Your E-mail address has been authorized to receive notifications for repository %s/%s. """ % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], - record.repository.namespace, record.repository.name, - record.repository.namespace, record.repository.name) + record.repository.namespace_user.username, record.repository.name, + record.repository.namespace_user.username, record.repository.name) return render_page_template('message.html', message=message) @web.route('/confirm', methods=['GET']) +@route_show_if(features.MAILING) def confirm_email(): code = request.values['code'] user = None new_email = None try: - user, new_email = model.confirm_user_email(code) + user, new_email, old_email = model.confirm_user_email(code) except model.DataModelException as ex: return render_page_template('confirmerror.html', error_message=ex.message) + if new_email: + send_email_changed(user.username, old_email, new_email) + common_login(user) return redirect(url_for('web.user', tab='email') diff --git a/initdb.py b/initdb.py index 7e48ae3af..6fa8efe98 100644 --- a/initdb.py +++ b/initdb.py @@ -51,7 +51,7 @@ def __gen_checksum(image_id): def __gen_image_id(repo, image_num): - str_to_hash = "%s/%s/%s" % (repo.namespace, repo.name, image_num) + str_to_hash = "%s/%s/%s" % (repo.namespace_user.username, repo.name, image_num) h = hashlib.md5(str_to_hash) return h.hexdigest() + h.hexdigest() @@ -79,12 +79,12 @@ def __create_subtree(repo, structure, creator_username, parent): creation_time = REFERENCE_DATE + timedelta(days=image_num) command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)] command = json.dumps(command_list) if command_list else None - new_image = model.set_image_metadata(docker_image_id, repo.namespace, - repo.name, str(creation_time), - 'no comment', command, parent) + new_image = model.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name, + str(creation_time), 'no comment', command, parent) - model.set_image_size(docker_image_id, repo.namespace, repo.name, - random.randrange(1, 1024 * 1024 * 1024)) + compressed_size = random.randrange(1, 1024 * 1024 * 1024) + model.set_image_size(docker_image_id, repo.namespace_user.username, repo.name, compressed_size, + int(compressed_size * 1.4)) # Populate the diff file diff_path = store.image_file_diffs_path(new_image.storage.uuid) @@ -100,7 +100,7 @@ def __create_subtree(repo, structure, creator_username, parent): last_node_tags = [last_node_tags] for tag_name in last_node_tags: - model.create_or_update_tag(repo.namespace, repo.name, tag_name, + model.create_or_update_tag(repo.namespace_user.username, repo.name, tag_name, new_image.docker_image_id) for subtree in subtrees: @@ -179,6 +179,8 @@ def initialize_database(): TeamRole.create(name='member') Visibility.create(name='public') Visibility.create(name='private') + + LoginService.create(name='google') LoginService.create(name='github') LoginService.create(name='quayrobot') LoginService.create(name='ldap') @@ -212,7 +214,11 @@ def initialize_database(): LogEntryKind.create(name='org_create_team') LogEntryKind.create(name='org_delete_team') + LogEntryKind.create(name='org_invite_team_member') + LogEntryKind.create(name='org_delete_team_member_invite') LogEntryKind.create(name='org_add_team_member') + LogEntryKind.create(name='org_team_member_invite_accepted') + LogEntryKind.create(name='org_team_member_invite_declined') LogEntryKind.create(name='org_remove_team_member') LogEntryKind.create(name='org_set_team_description') LogEntryKind.create(name='org_set_team_role') @@ -229,13 +235,15 @@ def initialize_database(): LogEntryKind.create(name='delete_application') LogEntryKind.create(name='reset_application_client_secret') - # Note: These are deprecated. + # Note: These next two are deprecated. LogEntryKind.create(name='add_repo_webhook') LogEntryKind.create(name='delete_repo_webhook') LogEntryKind.create(name='add_repo_notification') LogEntryKind.create(name='delete_repo_notification') + LogEntryKind.create(name='regenerate_robot_token') + ImageStorageLocation.create(name='local_eu') ImageStorageLocation.create(name='local_us') @@ -251,6 +259,10 @@ def initialize_database(): ExternalNotificationMethod.create(name='email') ExternalNotificationMethod.create(name='webhook') + ExternalNotificationMethod.create(name='flowdock') + ExternalNotificationMethod.create(name='hipchat') + ExternalNotificationMethod.create(name='slack') + NotificationKind.create(name='repo_push') NotificationKind.create(name='build_queued') NotificationKind.create(name='build_start') @@ -261,6 +273,7 @@ def initialize_database(): NotificationKind.create(name='over_private_usage') NotificationKind.create(name='expiring_license') NotificationKind.create(name='maintenance') + NotificationKind.create(name='org_team_invite') NotificationKind.create(name='test_notification') @@ -292,7 +305,7 @@ def populate_database(): new_user_2.verified = True new_user_2.save() - new_user_3 = model.create_user('freshuser', 'password', 'no@thanks.com') + new_user_3 = model.create_user('freshuser', 'password', 'jschorr+test@devtable.com') new_user_3.verified = True new_user_3.save() @@ -313,7 +326,8 @@ def populate_database(): outside_org.verified = True outside_org.save() - model.create_notification('test_notification', new_user_1, metadata={'some': 'value', 'arr': [1,2,3], 'obj': {'a': 1, 'b': 2}}) + model.create_notification('test_notification', new_user_1, + metadata={'some':'value', 'arr':[1, 2, 3], 'obj':{'a':1, 'b':2}}) from_date = datetime.utcnow() to_date = from_date + timedelta(hours=1) @@ -377,18 +391,20 @@ def populate_database(): }) trigger.save() - repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name) + repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username, building.name) job_config = { 'repository': repo, 'docker_tags': ['latest'], 'build_subdir': '', } - record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr@devtable.com') + record = model.create_email_authorization_for_repo(new_user_1.username, 'simple', + 'jschorr@devtable.com') record.confirmed = True record.save() - model.create_email_authorization_for_repo(new_user_1.username, 'simple', 'jschorr+other@devtable.com') + model.create_email_authorization_for_repo(new_user_1.username, 'simple', + 'jschorr+other@devtable.com') build2 = model.create_repository_build(building, token, job_config, '68daeebd-a5b9-457f-80a0-4363b882f8ea', @@ -415,12 +431,12 @@ def populate_database(): model.create_robot('coolrobot', org) - oauth.create_application(org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html', - client_id='deadbeef') + oauth.create_application(org, 'Some Test App', 'http://localhost:8000', + 'http://localhost:8000/o2c.html', client_id='deadbeef') - oauth.create_application(org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html', - client_id='deadpork', - description = 'This is another test application') + oauth.create_application(org, 'Some Other Test App', 'http://quay.io', + 'http://localhost:8000/o2c.html', client_id='deadpork', + description='This is another test application') model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin') @@ -442,8 +458,8 @@ def populate_database(): reader_team = model.create_team('readers', org, 'member', 'Readers of orgrepo.') - model.set_team_repo_permission(reader_team.name, org_repo.namespace, - org_repo.name, 'read') + model.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name, + 'read') model.add_user_to_team(new_user_2, reader_team) model.add_user_to_team(reader, reader_team) @@ -465,12 +481,9 @@ def populate_database(): (2, [], 'latest17'), (2, [], 'latest18'),]) - model.add_prototype_permission(org, 'read', activating_user=new_user_1, - delegate_user=new_user_2) - model.add_prototype_permission(org, 'read', activating_user=new_user_1, - delegate_team=reader_team) - model.add_prototype_permission(org, 'write', activating_user=new_user_2, - delegate_user=new_user_1) + model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_user=new_user_2) + model.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_team=reader_team) + model.add_prototype_permission(org, 'write', activating_user=new_user_2, delegate_user=new_user_1) today = datetime.today() week_ago = today - timedelta(6) diff --git a/license.py b/license.py deleted file mode 100644 index b45d90cf8..000000000 --- a/license.py +++ /dev/null @@ -1,13 +0,0 @@ -import pickle - -from Crypto.PublicKey import RSA - -n = 24311791124264168943780535074639421876317270880681911499019414944027362498498429776192966738844514582251884695124256895677070273097239290537016363098432785034818859765271229653729724078304186025013011992335454557504431888746007324285000011384941749613875855493086506022340155196030616409545906383713728780211095701026770053812741971198465120292345817928060114890913931047021503727972067476586739126160044293621653486418983183727572502888923949587290840425930251185737996066354726953382305020440374552871209809125535533731995494145421279907938079885061852265339259634996180877443852561265066616143910755505151318370667L -e = 65537L - -def load_license(license_path): - decryptor = RSA.construct((n, e)) - with open(license_path, 'rb') as encrypted_license: - decrypted_data = decryptor.encrypt(encrypted_license.read(), 0) - - return pickle.loads(decrypted_data[0]) diff --git a/license.pyc b/license.pyc deleted file mode 100644 index 83687adfa..000000000 Binary files a/license.pyc and /dev/null differ diff --git a/requirements-nover.txt b/requirements-nover.txt index c0979629b..a3c74e89b 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -32,5 +32,7 @@ raven python-ldap pycrypto logentries +psycopg2 +pyyaml git+https://github.com/DevTable/aniso8601-fake.git git+https://github.com/DevTable/anunidecode.git diff --git a/requirements.txt b/requirements.txt index 090ade690..e454e6846 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ Pillow==2.5.1 PyGithub==1.25.0 PyMySQL==0.6.2 PyPDF2==1.22 +PyYAML==3.11 SQLAlchemy==0.9.7 Werkzeug==0.9.6 alembic==0.6.5 @@ -44,6 +45,7 @@ python-dateutil==2.2 python-ldap==2.4.15 python-magic==0.4.6 pytz==2014.4 +psycopg2==2.5.3 raven==5.0.0 redis==2.10.1 reportlab==2.7 diff --git a/static/css/quay.css b/static/css/quay.css index 01fe84e60..5ec050200 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -21,8 +21,7 @@ #quay-logo { - width: 80px; - margin-right: 30px; + width: 100px; } #padding-container { @@ -145,6 +144,15 @@ nav.navbar-default .navbar-nav>li>a.active { max-width: 320px; } +.notification-view-element .right-controls button { + margin-left: 10px; +} + +.notification-view-element .message i.fa { + margin-right: 6px; +} + + .notification-view-element .orginfo { margin-top: 8px; float: left; @@ -464,6 +472,22 @@ i.toggle-icon:hover { .docker-auth-dialog .token-dialog-body .well { margin-bottom: 0px; + position: relative; + padding-right: 24px; +} + +.docker-auth-dialog .token-dialog-body .well i.fa-refresh { + position: absolute; + top: 9px; + right: 9px; + font-size: 20px; + color: gray; + transition: all 0.5s ease-in-out; + cursor: pointer; +} + +.docker-auth-dialog .token-dialog-body .well i.fa-refresh:hover { + color: black; } .docker-auth-dialog .token-view { @@ -729,7 +753,7 @@ i.toggle-icon:hover { } .user-notification.notification-animated { - width: 21px; + min-width: 21px; transform: scale(0); -moz-transform: scale(0); @@ -2257,6 +2281,14 @@ p.editable:hover i { position: relative; } +.copy-box-element.disabled .input-group-addon { + display: none; +} + +.copy-box-element.disabled input { + border-radius: 4px !important; +} + .global-zeroclipboard-container embed { cursor: pointer; } @@ -2535,7 +2567,7 @@ p.editable:hover i { margin-top: 10px; } -.repo-build .build-log-error-element { +.repo-build .build-log-error-element .error-message-container { position: relative; display: inline-block; margin: 10px; @@ -2545,7 +2577,7 @@ p.editable:hover i { margin-left: 22px; } -.repo-build .build-log-error-element i.fa { +.repo-build .build-log-error-element .error-message-container i.fa { color: red; position: absolute; top: 13px; @@ -3535,6 +3567,12 @@ p.editable:hover i { white-space: nowrap; } +.tt-message { + padding: 10px; + font-size: 12px; + white-space: nowrap; +} + .tt-suggestion p { margin: 0; } @@ -4226,7 +4264,7 @@ pre.command:before { } .user-row.super-user td { - background-color: #d9edf7; + background-color: #eeeeee; } .user-row .user-class { @@ -4559,6 +4597,27 @@ i.quay-icon { height: 16px; } +i.flowdock-icon { + background-image: url(/static/img/flowdock.ico); + background-size: 16px; + width: 16px; + height: 16px; +} + +i.hipchat-icon { + background-image: url(/static/img/hipchat.png); + background-size: 16px; + width: 16px; + height: 16px; +} + +i.slack-icon { + background-image: url(/static/img/slack.ico); + background-size: 16px; + width: 16px; + height: 16px; +} + .external-notification-view-element { margin: 10px; padding: 6px; @@ -4593,4 +4652,68 @@ i.quay-icon { .external-notification-view-element:hover .side-controls button { border: 1px solid #eee; +} + +.member-listing { + width: 100%; +} + +.member-listing .section-header { + color: #ccc; + margin-top: 20px; + margin-bottom: 10px; +} + +.member-listing .gravatar { + vertical-align: middle; + margin-right: 10px; +} + +.member-listing .entity-reference { + margin-bottom: 10px; + display: inline-block; +} + +.member-listing .invite-listing { + margin-bottom: 10px; + display: inline-block; +} + +.team-view .organization-header .popover { + max-width: none !important; +} + +.team-view .organization-header .popover.bottom-right .arrow:after { + border-bottom-color: #f7f7f7; + top: 2px; +} + +.team-view .organization-header .popover-content { + font-size: 14px; + padding-top: 6px; +} + +.team-view .organization-header .popover-content input { + background: white; +} + +.team-view .team-view-add-element .help-text { + font-size: 13px; + color: #ccc; + margin-top: 10px; +} + +.team-view .organization-header .popover-content { + min-width: 500px; +} + +#startTriggerDialog .trigger-description { + margin-bottom: 20px; + padding-bottom: 20px; + border-bottom: 1px solid #eee; +} + +#startTriggerDialog #runForm .field-title { + width: 120px; + padding-right: 10px; } \ No newline at end of file diff --git a/static/directives/build-log-error.html b/static/directives/build-log-error.html index 095f8edd0..13b399bb9 100644 --- a/static/directives/build-log-error.html +++ b/static/directives/build-log-error.html @@ -1,4 +1,23 @@ - - - - +
+ + + + + caused by attempting to pull private repository {{ getLocalPullInfo().repo }} + with inaccessible crdentials + without credentials + + + + +
+
+ Note: The credentials {{ getLocalPullInfo().login.username }} for registry {{ getLocalPullInfo().login.registry }} cannot + access repository {{ getLocalPullInfo().repo }}. +
+
+ Note: No robot account is specified for this build. Without such credentials, this pull will always fail. Please setup a new + build trigger with a robot account that has access to {{ getLocalPullInfo().repo }} or make that repository public. +
+
+
diff --git a/static/directives/copy-box.html b/static/directives/copy-box.html index 1d996cc31..07dea7407 100644 --- a/static/directives/copy-box.html +++ b/static/directives/copy-box.html @@ -1,4 +1,4 @@ -
+
diff --git a/static/directives/create-external-notification-dialog.html b/static/directives/create-external-notification-dialog.html index d384f3f59..bf0c5da03 100644 --- a/static/directives/create-external-notification-dialog.html +++ b/static/directives/create-external-notification-dialog.html @@ -73,7 +73,7 @@
- {{ field.title }}: + {{ field.title }}:
@@ -86,7 +86,11 @@ current-entity="currentConfig[field.name]" ng-model="currentConfig[field.name]" allowed-entities="['user', 'team', 'org']" - ng-switch-when="entity"> + ng-switch-when="entity">
+ +
diff --git a/static/directives/docker-auth-dialog.html b/static/directives/docker-auth-dialog.html index dcb71a25b..e45b8967d 100644 --- a/static/directives/docker-auth-dialog.html +++ b/static/directives/docker-auth-dialog.html @@ -10,19 +10,33 @@
The docker username is {{ username }} and the password is the token below. You may use any value for email.
-
+ +
+ Regenerating Token... + +
+ +
+
-
+
+ +
+
Download .dockercfg file -
diff --git a/static/directives/dropdown-select.html b/static/directives/dropdown-select.html index c1157e3d0..69404e161 100644 --- a/static/directives/dropdown-select.html +++ b/static/directives/dropdown-select.html @@ -2,7 +2,7 @@
+ ng-readonly="!allowCustomInput">
-
@@ -37,15 +37,7 @@ {{ user.username }} - - {{ notificationService.notifications.length }} - +
    @@ -58,11 +50,7 @@ Notifications - - {{ notificationService.notifications.length }} - +
  • Organizations
    • diff --git a/static/directives/logs-view.html b/static/directives/logs-view.html index 5ccea2f3f..28509b897 100644 --- a/static/directives/logs-view.html +++ b/static/directives/logs-view.html @@ -3,7 +3,7 @@
    Usage Logs - + From diff --git a/static/directives/manual-trigger-build-dialog.html b/static/directives/manual-trigger-build-dialog.html new file mode 100644 index 000000000..e82879e8a --- /dev/null +++ b/static/directives/manual-trigger-build-dialog.html @@ -0,0 +1,38 @@ + + \ No newline at end of file diff --git a/static/directives/notification-bar.html b/static/directives/notification-bar.html index 5d25a40b4..c6841a7f5 100644 --- a/static/directives/notification-bar.html +++ b/static/directives/notification-bar.html @@ -3,7 +3,10 @@
    -

    Notifications

    +

    + Notifications + +

    diff --git a/static/directives/notification-view.html b/static/directives/notification-view.html index f03133e55..fbdb4b419 100644 --- a/static/directives/notification-view.html +++ b/static/directives/notification-view.html @@ -7,10 +7,13 @@ {{ notification.organization }}
    -
    {{ parseDate(notification.created) | date:'medium'}}
    Dismiss Notification +
    +
    {{ parseDate(notification.created) | date:'medium'}}
    diff --git a/static/directives/notifications-bubble.html b/static/directives/notifications-bubble.html new file mode 100644 index 000000000..cf10cccf2 --- /dev/null +++ b/static/directives/notifications-bubble.html @@ -0,0 +1,7 @@ + + + {{ notificationService.notifications.length }}+ + + diff --git a/static/directives/robots-manager.html b/static/directives/robots-manager.html index c696937d2..c11c07cf8 100644 --- a/static/directives/robots-manager.html +++ b/static/directives/robots-manager.html @@ -31,7 +31,7 @@
    + shown="!!shownRobot" counter="showRobotCounter" supports-regenerate="true" regenerate="regenerateToken(username)"> {{ shownRobot.name }}
diff --git a/static/directives/signin-form.html b/static/directives/signin-form.html index f56b8f8db..59551b51c 100644 --- a/static/directives/signin-form.html +++ b/static/directives/signin-form.html @@ -1,21 +1,27 @@