diff --git a/Dockerfile b/Dockerfile index 6b7cbf557..90a17c9fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 24JUN2015 +RUN apt-get update # 22OCT2015 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev libgpgme11 libgpgme11-dev diff --git a/README.md b/README.md index e73ec0cfd..a5e9ea86a 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ To build and run a docker container, pass one argument to local-docker.sh: - `buildman`: run the buildmanager - `notifications`: run the notification worker - `test`: run the unit tests +- `initdb`: clear and initialize the test database For example: diff --git a/app.py b/app.py index 0bf65d5e7..cb90050ec 100644 --- a/app.py +++ b/app.py @@ -151,12 +151,11 @@ dex_login = DexOAuthConfig(app.config, 'DEX_LOGIN_CONFIG') oauth_apps = [github_login, github_trigger, gitlab_trigger, google_login, dex_login] -image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf, metric_queue=metric_queue) -image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf, metric_queue=metric_queue) +image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf) +image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, - metric_queue=metric_queue, reporter=MetricQueueReporter(metric_queue)) -notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf, metric_queue=metric_queue) +notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) diff --git a/buildman/builder.py b/buildman/builder.py index 8f1bb18c5..08f5f733a 100644 --- a/buildman/builder.py +++ b/buildman/builder.py @@ -2,6 +2,7 @@ import logging import os import features import time +import socket from app import app, userfiles as user_files, build_logs, dockerfile_build_queue @@ -10,6 +11,8 @@ from buildman.manager.ephemeral import EphemeralBuilderManager from buildman.server import BuilderServer from trollius import SSLContext +from raven.handlers.logging import SentryHandler +from raven.conf import setup_logging logger = logging.getLogger(__name__) @@ -77,4 +80,10 @@ if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) logging.getLogger('peewee').setLevel(logging.WARN) logging.getLogger('boto').setLevel(logging.WARN) + + if app.config.get('EXCEPTION_LOG_TYPE', 'FakeSentry') == 'Sentry': + buildman_name = '%s:buildman' % socket.gethostname() + setup_logging(SentryHandler(app.config.get('SENTRY_DSN', ''), name=buildman_name, + level=logging.ERROR)) + run_build_manager() diff --git a/buildman/jobutil/buildjob.py b/buildman/jobutil/buildjob.py index f6291f62b..dbbb8113f 100644 --- a/buildman/jobutil/buildjob.py +++ b/buildman/jobutil/buildjob.py @@ -128,10 +128,9 @@ class BuildJob(object): return False full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step] - logger.debug('Checking step #%s: %s, %s == %s', step, image.id, - image.storage.command, full_command) + logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command) - return image.storage.command == full_command + return image.command == full_command path = tree.find_longest_path(base_image.id, checker) if not path: diff --git a/buildman/jobutil/buildstatus.py b/buildman/jobutil/buildstatus.py index dfb97bb40..079615812 100644 --- a/buildman/jobutil/buildstatus.py +++ b/buildman/jobutil/buildstatus.py @@ -1,6 +1,11 @@ from data.database import BUILD_PHASE from data import model +from redis import RedisError + import datetime +import logging + +logger = logging.getLogger(__name__) class StatusHandler(object): """ Context wrapper for writing status to build logs. """ @@ -24,7 +29,11 @@ class StatusHandler(object): def _append_log_message(self, log_message, log_type=None, log_data=None): log_data = log_data or {} log_data['datetime'] = str(datetime.datetime.now()) - self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data) + + try: + self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data) + except RedisError: + logger.exception('Could not save build log for build %s: %s', self._uuid, log_message) def append_log(self, log_message, extra_data=None): if log_message is None: @@ -64,4 +73,7 @@ class StatusHandler(object): return self._status def __exit__(self, exc_type, value, traceback): - self._build_logs.set_status(self._uuid, self._status) + try: + self._build_logs.set_status(self._uuid, self._status) + except RedisError: + logger.exception('Could not set status of build %s to %s', self._uuid, self._status) diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py index 0d1c482e2..eb5dd13f7 100644 --- a/buildman/manager/ephemeral.py +++ b/buildman/manager/ephemeral.py @@ -10,6 +10,7 @@ from trollius import From, coroutine, Return, async from concurrent.futures import ThreadPoolExecutor from urllib3.exceptions import ReadTimeoutError, ProtocolError +from app import metric_queue from buildman.manager.basemanager import BaseManager from buildman.manager.executor import PopenExecutor, EC2Executor from buildman.component.buildcomponent import BuildComponent @@ -98,6 +99,10 @@ class EphemeralBuilderManager(BaseManager): if restarter is not None: async(restarter()) + except (KeyError, etcd.EtcdKeyError): + logger.debug('Etcd key already cleared: %s', etcd_key) + return + except etcd.EtcdException as eex: # TODO(jschorr): This is a quick and dirty hack and should be replaced # with a proper exception check. @@ -335,6 +340,7 @@ class EphemeralBuilderManager(BaseManager): try: builder_id = yield From(self._executor.start_builder(realm, token, build_uuid)) + metric_queue.put('EC2BuilderStarted', 1, unit='Count') except: logger.exception('Exception when starting builder for job: %s', build_uuid) raise Return(False, EC2_API_TIMEOUT) @@ -399,7 +405,7 @@ class EphemeralBuilderManager(BaseManager): try: yield From(self._etcd_client.delete(job_key)) except (KeyError, etcd.EtcdKeyError): - logger.exception('Builder is asking for job to be removed, but work already completed') + logger.debug('Builder is asking for job to be removed, but work already completed') self.job_complete_callback(build_job, job_status) diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py index 449d66ed3..e4f9fb7bb 100644 --- a/buildman/manager/executor.py +++ b/buildman/manager/executor.py @@ -160,8 +160,17 @@ class EC2Executor(BuilderExecutor): @coroutine def stop_builder(self, builder_id): - ec2_conn = self._get_conn() - terminated_instances = yield From(ec2_conn.terminate_instances([builder_id])) + try: + ec2_conn = self._get_conn() + terminated_instances = yield From(ec2_conn.terminate_instances([builder_id])) + except boto.exception.EC2ResponseError as ec2e: + if ec2e.error_code == 404: + logger.debug('Instance %s already terminated', builder_id) + return + + logger.exception('Exception when trying to terminate instance %s', builder_id) + raise + if builder_id not in [si.id for si in terminated_instances]: raise ExecutorException('Unable to terminate instance: %s' % builder_id) diff --git a/buildman/server.py b/buildman/server.py index f2a4feb92..80776f6e2 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -65,7 +65,7 @@ class BuilderServer(object): @controller_app.route('/status') def status(): - metrics = server._queue.get_metrics(require_transaction=False) + metrics = server._queue.get_metrics() (running_count, available_not_running_count, available_count) = metrics workers = [component for component in server._current_components diff --git a/buildtrigger/__init__.py b/buildtrigger/__init__.py new file mode 100644 index 000000000..8a794cf96 --- /dev/null +++ b/buildtrigger/__init__.py @@ -0,0 +1,5 @@ +import buildtrigger.bitbuckethandler +import buildtrigger.customhandler +import buildtrigger.githubhandler +import buildtrigger.gitlabhandler + diff --git a/buildtrigger/basehandler.py b/buildtrigger/basehandler.py new file mode 100644 index 000000000..2555b09ed --- /dev/null +++ b/buildtrigger/basehandler.py @@ -0,0 +1,222 @@ +from endpoints.building import PreparedBuild +from data import model +from buildtrigger.triggerutil import get_trigger_config, InvalidServiceException +from jsonschema import validate + +METADATA_SCHEMA = { + 'type': 'object', + 'properties': { + 'commit': { + 'type': 'string', + 'description': 'first 7 characters of the SHA-1 identifier for a git commit', + 'pattern': '^([A-Fa-f0-9]{7,})$', + }, + 'git_url': { + 'type': 'string', + 'description': 'The GIT url to use for the checkout', + }, + 'ref': { + 'type': 'string', + 'description': 'git reference for a git commit', + 'pattern': '^refs\/(heads|tags|remotes)\/(.+)$', + }, + 'default_branch': { + 'type': 'string', + 'description': 'default branch of the git repository', + }, + 'commit_info': { + 'type': 'object', + 'description': 'metadata about a git commit', + 'properties': { + 'url': { + 'type': 'string', + 'description': 'URL to view a git commit', + }, + 'message': { + 'type': 'string', + 'description': 'git commit message', + }, + 'date': { + 'type': 'string', + 'description': 'timestamp for a git commit' + }, + 'author': { + 'type': 'object', + 'description': 'metadata about the author of a git commit', + 'properties': { + 'username': { + 'type': 'string', + 'description': 'username of the author', + }, + 'url': { + 'type': 'string', + 'description': 'URL to view the profile of the author', + }, + 'avatar_url': { + 'type': 'string', + 'description': 'URL to view the avatar of the author', + }, + }, + 'required': ['username'], + }, + 'committer': { + 'type': 'object', + 'description': 'metadata about the committer of a git commit', + 'properties': { + 'username': { + 'type': 'string', + 'description': 'username of the committer', + }, + 'url': { + 'type': 'string', + 'description': 'URL to view the profile of the committer', + }, + 'avatar_url': { + 'type': 'string', + 'description': 'URL to view the avatar of the committer', + }, + }, + 'required': ['username'], + }, + }, + 'required': ['url', 'message', 'date'], + }, + }, + 'required': ['commit', 'git_url'], +} + + +class BuildTriggerHandler(object): + def __init__(self, trigger, override_config=None): + self.trigger = trigger + self.config = override_config or get_trigger_config(trigger) + + @property + def auth_token(self): + """ Returns the auth token for the trigger. """ + return self.trigger.auth_token + + def load_dockerfile_contents(self): + """ + Loads the Dockerfile found for the trigger's config and returns them or None if none could + be found/loaded. + """ + raise NotImplementedError + + def list_build_sources(self): + """ + Take the auth information for the specific trigger type and load the + list of build sources(repositories). + """ + raise NotImplementedError + + def list_build_subdirs(self): + """ + Take the auth information and the specified config so far and list all of + the possible subdirs containing dockerfiles. + """ + raise NotImplementedError + + def handle_trigger_request(self): + """ + Transform the incoming request data into a set of actions. Returns a PreparedBuild. + """ + raise NotImplementedError + + def is_active(self): + """ + Returns True if the current build trigger is active. Inactive means further + setup is needed. + """ + raise NotImplementedError + + def activate(self, standard_webhook_url): + """ + Activates the trigger for the service, with the given new configuration. + Returns new public and private config that should be stored if successful. + """ + raise NotImplementedError + + def deactivate(self): + """ + Deactivates the trigger for the service, removing any hooks installed in + the remote service. Returns the new config that should be stored if this + trigger is going to be re-activated. + """ + raise NotImplementedError + + def manual_start(self, run_parameters=None): + """ + Manually creates a repository build for this trigger. Returns a PreparedBuild. + """ + raise NotImplementedError + + def list_field_values(self, field_name, limit=None): + """ + Lists all values for the given custom trigger field. For example, a trigger might have a + field named "branches", and this method would return all branches. + """ + raise NotImplementedError + + def get_repository_url(self): + """ Returns the URL of the current trigger's repository. Note that this operation + can be called in a loop, so it should be as fast as possible. """ + raise NotImplementedError + + @classmethod + def service_name(cls): + """ + Particular service implemented by subclasses. + """ + raise NotImplementedError + + @classmethod + def get_handler(cls, trigger, override_config=None): + for subc in cls.__subclasses__(): + if subc.service_name() == trigger.service.name: + return subc(trigger, override_config) + + raise InvalidServiceException('Unable to find service: %s' % trigger.service.name) + + def put_config_key(self, key, value): + """ Updates a config key in the trigger, saving it to the DB. """ + self.config[key] = value + model.build.update_build_trigger(self.trigger, self.config) + + def set_auth_token(self, auth_token): + """ Sets the auth token for the trigger, saving it to the DB. """ + model.build.update_build_trigger(self.trigger, self.config, auth_token=auth_token) + + def get_dockerfile_path(self): + """ Returns the normalized path to the Dockerfile found in the subdirectory + in the config. """ + subdirectory = self.config.get('subdir', '') + if subdirectory == '/': + subdirectory = '' + else: + if not subdirectory.endswith('/'): + subdirectory = subdirectory + '/' + + return subdirectory + 'Dockerfile' + + def prepare_build(self, metadata, is_manual=False): + # Ensure that the metadata meets the scheme. + validate(metadata, METADATA_SCHEMA) + + config = self.config + ref = metadata.get('ref', None) + commit_sha = metadata['commit'] + default_branch = metadata.get('default_branch', None) + + prepared = PreparedBuild(self.trigger) + prepared.name_from_sha(commit_sha) + prepared.subdirectory = config.get('subdir', None) + prepared.is_manual = is_manual + prepared.metadata = metadata + + if ref is not None: + prepared.tags_from_ref(ref, default_branch) + else: + prepared.tags = [commit_sha[:7]] + + return prepared diff --git a/buildtrigger/bitbuckethandler.py b/buildtrigger/bitbuckethandler.py new file mode 100644 index 000000000..e14d75b09 --- /dev/null +++ b/buildtrigger/bitbuckethandler.py @@ -0,0 +1,549 @@ +import logging +import re + +from jsonschema import validate +from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, + TriggerDeactivationException, TriggerStartException, + InvalidPayloadException, + determine_build_ref, raise_if_skipped_build, + find_matching_branches) + +from buildtrigger.basehandler import BuildTriggerHandler + +from app import app, get_app_url +from bitbucket import BitBucket +from util.security.ssh import generate_ssh_keypair +from util.dict_wrappers import JSONPathDict, SafeDictSetter + +logger = logging.getLogger(__name__) + +_BITBUCKET_COMMIT_URL = 'https://bitbucket.org/%s/commits/%s' +_RAW_AUTHOR_REGEX = re.compile(r'.*<(.+)>') + +BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA = { + 'type': 'object', + 'properties': { + 'repository': { + 'type': 'object', + 'properties': { + 'full_name': { + 'type': 'string', + }, + }, + 'required': ['full_name'], + }, + 'push': { + 'type': 'object', + 'properties': { + 'changes': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'new': { + 'type': 'object', + 'properties': { + 'target': { + 'type': 'object', + 'properties': { + 'hash': { + 'type': 'string' + }, + 'message': { + 'type': 'string' + }, + 'date': { + 'type': 'string' + }, + 'author': { + 'type': 'object', + 'properties': { + 'user': { + 'type': 'object', + 'properties': { + 'username': { + 'type': 'string', + }, + 'links': { + 'type': 'object', + 'properties': { + 'html': { + 'type': 'object', + 'properties': { + 'href': { + 'type': 'string', + }, + }, + 'required': ['href'], + }, + 'avatar': { + 'type': 'object', + 'properties': { + 'href': { + 'type': 'string', + }, + }, + 'required': ['href'], + }, + }, + 'required': ['html', 'avatar'], + }, + }, + 'required': ['username'], + }, + }, + }, + 'links': { + 'type': 'object', + 'properties': { + 'html': { + 'type': 'object', + 'properties': { + 'href': { + 'type': 'string', + }, + }, + 'required': ['href'], + }, + }, + 'required': ['html'], + }, + }, + 'required': ['hash', 'message', 'date'], + }, + }, + 'required': ['target'], + }, + }, + }, + }, + }, + 'required': ['changes'], + }, + }, + 'actor': { + 'type': 'object', + 'properties': { + 'username': { + 'type': 'string', + }, + 'links': { + 'type': 'object', + 'properties': { + 'html': { + 'type': 'object', + 'properties': { + 'href': { + 'type': 'string', + }, + }, + 'required': ['href'], + }, + 'avatar': { + 'type': 'object', + 'properties': { + 'href': { + 'type': 'string', + }, + }, + 'required': ['href'], + }, + }, + 'required': ['html', 'avatar'], + }, + }, + 'required': ['username'], + }, + 'required': ['push', 'repository'], +} + +BITBUCKET_COMMIT_INFO_SCHEMA = { + 'type': 'object', + 'properties': { + 'node': { + 'type': 'string', + }, + 'message': { + 'type': 'string', + }, + 'timestamp': { + 'type': 'string', + }, + 'raw_author': { + 'type': 'string', + }, + }, + 'required': ['node', 'message', 'timestamp'] +} + +def get_transformed_commit_info(bb_commit, ref, default_branch, repository_name, lookup_author): + """ Returns the BitBucket commit information transformed into our own + payload format. + """ + try: + validate(bb_commit, BITBUCKET_COMMIT_INFO_SCHEMA) + except Exception as exc: + logger.exception('Exception when validating Bitbucket commit information: %s from %s', exc.message, bb_commit) + raise InvalidPayloadException(exc.message) + + commit = JSONPathDict(bb_commit) + + config = SafeDictSetter() + config['commit'] = commit['node'] + config['ref'] = ref + config['default_branch'] = default_branch + config['git_url'] = 'git@bitbucket.org:%s.git' % repository_name + + config['commit_info.url'] = _BITBUCKET_COMMIT_URL % (repository_name, commit['node']) + config['commit_info.message'] = commit['message'] + config['commit_info.date'] = commit['timestamp'] + + match = _RAW_AUTHOR_REGEX.match(commit['raw_author']) + if match: + email_address = match.group(1) + author_info = JSONPathDict(lookup_author(email_address)) + if author_info: + config['commit_info.author.username'] = author_info['user.username'] + config['commit_info.author.url'] = 'https://bitbucket.org/%s/' % author_info['user.username'] + config['commit_info.author.avatar_url'] = author_info['user.avatar'] + + return config.dict_value() + + +def get_transformed_webhook_payload(bb_payload, default_branch=None): + """ Returns the BitBucket webhook JSON payload transformed into our own payload + format. If the bb_payload is not valid, returns None. + """ + try: + validate(bb_payload, BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA) + except Exception as exc: + logger.exception('Exception when validating Bitbucket webhook payload: %s from %s', exc.message, bb_payload) + raise InvalidPayloadException(exc.message) + + payload = JSONPathDict(bb_payload) + change = payload['push.changes[-1].new'] + if not change: + return None + + ref = ('refs/heads/' + change['name'] if change['type'] == 'branch' + else 'refs/tags/' + change['name']) + + repository_name = payload['repository.full_name'] + target = change['target'] + + config = SafeDictSetter() + config['commit'] = target['hash'] + config['ref'] = ref + config['default_branch'] = default_branch + config['git_url'] = 'git@bitbucket.org:%s.git' % repository_name + + config['commit_info.url'] = target['links.html.href'] + config['commit_info.message'] = target['message'] + config['commit_info.date'] = target['date'] + + config['commit_info.author.username'] = target['author.user.username'] + config['commit_info.author.url'] = target['author.user.links.html.href'] + config['commit_info.author.avatar_url'] = target['author.user.links.avatar.href'] + + config['commit_info.committer.username'] = payload['actor.username'] + config['commit_info.committer.url'] = payload['actor.links.html.href'] + config['commit_info.committer.avatar_url'] = payload['actor.links.avatar.href'] + return config.dict_value() + + +class BitbucketBuildTrigger(BuildTriggerHandler): + """ + BuildTrigger for Bitbucket. + """ + @classmethod + def service_name(cls): + return 'bitbucket' + + def _get_client(self): + """ Returns a BitBucket API client for this trigger's config. """ + key = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_KEY', '') + secret = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_SECRET', '') + + trigger_uuid = self.trigger.uuid + callback_url = '%s/oauth1/bitbucket/callback/trigger/%s' % (get_app_url(), trigger_uuid) + + return BitBucket(key, secret, callback_url, timeout=5) + + def _get_authorized_client(self): + """ Returns an authorized API client. """ + base_client = self._get_client() + auth_token = self.auth_token or 'invalid:invalid' + token_parts = auth_token.split(':') + if len(token_parts) != 2: + token_parts = ['invalid', 'invalid'] + + (access_token, access_token_secret) = token_parts + return base_client.get_authorized_client(access_token, access_token_secret) + + def _get_repository_client(self): + """ Returns an API client for working with this config's BB repository. """ + source = self.config['build_source'] + (namespace, name) = source.split('/') + bitbucket_client = self._get_authorized_client() + return bitbucket_client.for_namespace(namespace).repositories().get(name) + + def _get_default_branch(self, repository, default_value='master'): + """ Returns the default branch for the repository or the value given. """ + (result, data, _) = repository.get_main_branch() + if result: + return data['name'] + + return default_value + + def get_oauth_url(self): + """ Returns the OAuth URL to authorize Bitbucket. """ + bitbucket_client = self._get_client() + (result, data, err_msg) = bitbucket_client.get_authorization_url() + if not result: + raise TriggerProviderException(err_msg) + + return data + + def exchange_verifier(self, verifier): + """ Exchanges the given verifier token to setup this trigger. """ + bitbucket_client = self._get_client() + access_token = self.config.get('access_token', '') + access_token_secret = self.auth_token + + # Exchange the verifier for a new access token. + (result, data, _) = bitbucket_client.verify_token(access_token, access_token_secret, verifier) + if not result: + return False + + # Save the updated access token and secret. + self.set_auth_token(data[0] + ':' + data[1]) + + # Retrieve the current authorized user's information and store the username in the config. + authorized_client = self._get_authorized_client() + (result, data, _) = authorized_client.get_current_user() + if not result: + return False + + username = data['user']['username'] + self.put_config_key('username', username) + return True + + def is_active(self): + return 'webhook_id' in self.config + + def activate(self, standard_webhook_url): + config = self.config + + # Add a deploy key to the repository. + public_key, private_key = generate_ssh_keypair() + config['credentials'] = [ + { + 'name': 'SSH Public Key', + 'value': public_key, + }, + ] + + repository = self._get_repository_client() + (result, created_deploykey, err_msg) = repository.deploykeys().create( + app.config['REGISTRY_TITLE'] + ' webhook key', public_key) + + if not result: + msg = 'Unable to add deploy key to repository: %s' % err_msg + raise TriggerActivationException(msg) + + config['deploy_key_id'] = created_deploykey['pk'] + + # Add a webhook callback. + description = 'Webhook for invoking builds on %s' % app.config['REGISTRY_TITLE_SHORT'] + webhook_events = ['repo:push'] + (result, created_webhook, err_msg) = repository.webhooks().create( + description, standard_webhook_url, webhook_events) + + if not result: + msg = 'Unable to add webhook to repository: %s' % err_msg + raise TriggerActivationException(msg) + + config['webhook_id'] = created_webhook['uuid'] + self.config = config + return config, {'private_key': private_key} + + def deactivate(self): + config = self.config + + webhook_id = config.pop('webhook_id', None) + deploy_key_id = config.pop('deploy_key_id', None) + repository = self._get_repository_client() + + # Remove the webhook. + if webhook_id is not None: + (result, _, err_msg) = repository.webhooks().delete(webhook_id) + if not result: + msg = 'Unable to remove webhook from repository: %s' % err_msg + raise TriggerDeactivationException(msg) + + # Remove the public key. + if deploy_key_id is not None: + (result, _, err_msg) = repository.deploykeys().delete(deploy_key_id) + if not result: + msg = 'Unable to remove deploy key from repository: %s' % err_msg + raise TriggerDeactivationException(msg) + + return config + + def list_build_sources(self): + bitbucket_client = self._get_authorized_client() + (result, data, err_msg) = bitbucket_client.get_visible_repositories() + if not result: + raise RepositoryReadException('Could not read repository list: ' + err_msg) + + namespaces = {} + for repo in data: + if not repo['scm'] == 'git': + continue + + owner = repo['owner'] + if not owner in namespaces: + namespaces[owner] = { + 'personal': owner == self.config.get('username'), + 'repos': [], + 'info': { + 'name': owner + } + } + + namespaces[owner]['repos'].append(owner + '/' + repo['slug']) + + return namespaces.values() + + def list_build_subdirs(self): + config = self.config + repository = self._get_repository_client() + + # Find the first matching branch. + repo_branches = self.list_field_values('branch_name') or [] + branches = find_matching_branches(config, repo_branches) + if not branches: + branches = [self._get_default_branch(repository)] + + (result, data, err_msg) = repository.get_path_contents('', revision=branches[0]) + if not result: + raise RepositoryReadException(err_msg) + + files = set([f['path'] for f in data['files']]) + if 'Dockerfile' in files: + return ['/'] + + return [] + + def load_dockerfile_contents(self): + repository = self._get_repository_client() + path = self.get_dockerfile_path() + + (result, data, err_msg) = repository.get_raw_path_contents(path, revision='master') + if not result: + raise RepositoryReadException(err_msg) + + return data + + def list_field_values(self, field_name, limit=None): + source = self.config['build_source'] + (namespace, name) = source.split('/') + + bitbucket_client = self._get_authorized_client() + repository = bitbucket_client.for_namespace(namespace).repositories().get(name) + + if field_name == 'refs': + (result, data, _) = repository.get_branches_and_tags() + if not result: + return None + + branches = [b['name'] for b in data['branches']] + tags = [t['name'] for t in data['tags']] + + return ([{'kind': 'branch', 'name': b} for b in branches] + + [{'kind': 'tag', 'name': tag} for tag in tags]) + + if field_name == 'tag_name': + (result, data, _) = repository.get_tags() + if not result: + return None + + tags = list(data.keys()) + if limit: + tags = tags[0:limit] + + return tags + + if field_name == 'branch_name': + (result, data, _) = repository.get_branches() + if not result: + return None + + branches = list(data.keys()) + if limit: + branches = branches[0:limit] + + return branches + + return None + + def get_repository_url(self): + source = self.config['build_source'] + (namespace, name) = source.split('/') + return 'https://bitbucket.org/%s/%s' % (namespace, name) + + def handle_trigger_request(self, request): + payload = request.get_json() + logger.debug('Got BitBucket request: %s', payload) + + repository = self._get_repository_client() + default_branch = self._get_default_branch(repository) + + metadata = get_transformed_webhook_payload(payload, default_branch=default_branch) + prepared = self.prepare_build(metadata) + + # Check if we should skip this build. + raise_if_skipped_build(prepared, self.config) + return prepared + + def manual_start(self, run_parameters=None): + run_parameters = run_parameters or {} + repository = self._get_repository_client() + bitbucket_client = self._get_authorized_client() + + def get_branch_sha(branch_name): + # Lookup the commit SHA for the branch. + (result, data, _) = repository.get_branches() + if not result or not branch_name in data: + raise TriggerStartException('Could not find branch commit SHA') + + return data[branch_name]['node'] + + def get_tag_sha(tag_name): + # Lookup the commit SHA for the tag. + (result, data, _) = repository.get_tags() + if not result or not tag_name in data: + raise TriggerStartException('Could not find tag commit SHA') + + return data[tag_name]['node'] + + def lookup_author(email_address): + (result, data, _) = bitbucket_client.accounts().get_profile(email_address) + return data if result else None + + # Find the branch or tag to build. + default_branch = self._get_default_branch(repository) + (commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, + default_branch) + + # Lookup the commit SHA in BitBucket. + (result, commit_info, _) = repository.changesets().get(commit_sha) + if not result: + raise TriggerStartException('Could not lookup commit SHA') + + # Return a prepared build for the commit. + repository_name = '%s/%s' % (repository.namespace, repository.repository_name) + metadata = get_transformed_commit_info(commit_info, ref, default_branch, + repository_name, lookup_author) + + return self.prepare_build(metadata, is_manual=True) diff --git a/buildtrigger/customhandler.py b/buildtrigger/customhandler.py new file mode 100644 index 000000000..c60d751ea --- /dev/null +++ b/buildtrigger/customhandler.py @@ -0,0 +1,166 @@ +import logging +import json + +from jsonschema import validate +from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, + TriggerStartException, ValidationRequestException, + InvalidPayloadException, + SkipRequestException, raise_if_skipped_build, + find_matching_branches) + +from buildtrigger.basehandler import BuildTriggerHandler + +from util.security.ssh import generate_ssh_keypair + + +logger = logging.getLogger(__name__) + +class CustomBuildTrigger(BuildTriggerHandler): + payload_schema = { + 'type': 'object', + 'properties': { + 'commit': { + 'type': 'string', + 'description': 'first 7 characters of the SHA-1 identifier for a git commit', + 'pattern': '^([A-Fa-f0-9]{7,})$', + }, + 'ref': { + 'type': 'string', + 'description': 'git reference for a git commit', + 'pattern': '^refs\/(heads|tags|remotes)\/(.+)$', + }, + 'default_branch': { + 'type': 'string', + 'description': 'default branch of the git repository', + }, + 'commit_info': { + 'type': 'object', + 'description': 'metadata about a git commit', + 'properties': { + 'url': { + 'type': 'string', + 'description': 'URL to view a git commit', + }, + 'message': { + 'type': 'string', + 'description': 'git commit message', + }, + 'date': { + 'type': 'string', + 'description': 'timestamp for a git commit' + }, + 'author': { + 'type': 'object', + 'description': 'metadata about the author of a git commit', + 'properties': { + 'username': { + 'type': 'string', + 'description': 'username of the author', + }, + 'url': { + 'type': 'string', + 'description': 'URL to view the profile of the author', + }, + 'avatar_url': { + 'type': 'string', + 'description': 'URL to view the avatar of the author', + }, + }, + 'required': ['username', 'url', 'avatar_url'], + }, + 'committer': { + 'type': 'object', + 'description': 'metadata about the committer of a git commit', + 'properties': { + 'username': { + 'type': 'string', + 'description': 'username of the committer', + }, + 'url': { + 'type': 'string', + 'description': 'URL to view the profile of the committer', + }, + 'avatar_url': { + 'type': 'string', + 'description': 'URL to view the avatar of the committer', + }, + }, + 'required': ['username', 'url', 'avatar_url'], + }, + }, + 'required': ['url', 'message', 'date'], + }, + }, + 'required': ['commit', 'ref', 'default_branch'], + } + + @classmethod + def service_name(cls): + return 'custom-git' + + def is_active(self): + return self.config.has_key('credentials') + + def _metadata_from_payload(self, payload): + try: + metadata = json.loads(payload) + validate(metadata, self.payload_schema) + except Exception as e: + raise InvalidPayloadException(e.message) + return metadata + + def handle_trigger_request(self, request): + payload = request.data + if not payload: + raise InvalidPayloadException() + + logger.debug('Payload %s', payload) + + metadata = self._metadata_from_payload(payload) + metadata['git_url'] = self.config['build_source'] + + prepared = self.prepare_build(metadata) + + # Check if we should skip this build. + raise_if_skipped_build(prepared, self.config) + + return prepared + + def manual_start(self, run_parameters=None): + # commit_sha is the only required parameter + commit_sha = run_parameters.get('commit_sha') + if commit_sha is None: + raise TriggerStartException('missing required parameter') + + config = self.config + metadata = { + 'commit': commit_sha, + 'git_url': config['build_source'], + } + + return self.prepare_build(metadata, is_manual=True) + + def activate(self, standard_webhook_url): + config = self.config + public_key, private_key = generate_ssh_keypair() + config['credentials'] = [ + { + 'name': 'SSH Public Key', + 'value': public_key, + }, + { + 'name': 'Webhook Endpoint URL', + 'value': standard_webhook_url, + }, + ] + self.config = config + return config, {'private_key': private_key} + + def deactivate(self): + config = self.config + config.pop('credentials', None) + self.config = config + return config + + def get_repository_url(self): + return None diff --git a/buildtrigger/githubhandler.py b/buildtrigger/githubhandler.py new file mode 100644 index 000000000..2423cbc07 --- /dev/null +++ b/buildtrigger/githubhandler.py @@ -0,0 +1,515 @@ +import logging +import os.path +import base64 + +from app import app, github_trigger +from jsonschema import validate + +from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, + TriggerDeactivationException, TriggerStartException, + EmptyRepositoryException, ValidationRequestException, + SkipRequestException, InvalidPayloadException, + determine_build_ref, raise_if_skipped_build, + find_matching_branches) + +from buildtrigger.basehandler import BuildTriggerHandler + +from util.security.ssh import generate_ssh_keypair +from util.dict_wrappers import JSONPathDict, SafeDictSetter + +from github import (Github, UnknownObjectException, GithubException, + BadCredentialsException as GitHubBadCredentialsException) + +logger = logging.getLogger(__name__) + +GITHUB_WEBHOOK_PAYLOAD_SCHEMA = { + 'type': 'object', + 'properties': { + 'ref': { + 'type': 'string', + }, + 'head_commit': { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + }, + 'url': { + 'type': 'string', + }, + 'message': { + 'type': 'string', + }, + 'timestamp': { + 'type': 'string', + }, + 'author': { + 'type': 'object', + 'properties': { + 'username': { + 'type': 'string' + }, + 'html_url': { + 'type': 'string' + }, + 'avatar_url': { + 'type': 'string' + }, + }, + }, + 'committer': { + 'type': 'object', + 'properties': { + 'username': { + 'type': 'string' + }, + 'html_url': { + 'type': 'string' + }, + 'avatar_url': { + 'type': 'string' + }, + }, + }, + }, + 'required': ['id', 'url', 'message', 'timestamp'], + }, + 'repository': { + 'type': 'object', + 'properties': { + 'ssh_url': { + 'type': 'string', + }, + }, + 'required': ['ssh_url'], + }, + }, + 'required': ['ref', 'head_commit', 'repository'], +} + +def get_transformed_webhook_payload(gh_payload, default_branch=None, lookup_user=None): + """ Returns the GitHub webhook JSON payload transformed into our own payload + format. If the gh_payload is not valid, returns None. + """ + try: + validate(gh_payload, GITHUB_WEBHOOK_PAYLOAD_SCHEMA) + except Exception as exc: + raise InvalidPayloadException(exc.message) + + payload = JSONPathDict(gh_payload) + + config = SafeDictSetter() + config['commit'] = payload['head_commit.id'] + config['ref'] = payload['ref'] + config['default_branch'] = default_branch + config['git_url'] = payload['repository.ssh_url'] + + config['commit_info.url'] = payload['head_commit.url'] + config['commit_info.message'] = payload['head_commit.message'] + config['commit_info.date'] = payload['head_commit.timestamp'] + + config['commit_info.author.username'] = payload['head_commit.author.username'] + config['commit_info.author.url'] = payload.get('head_commit.author.html_url') + config['commit_info.author.avatar_url'] = payload.get('head_commit.author.avatar_url') + + config['commit_info.committer.username'] = payload.get('head_commit.committer.username') + config['commit_info.committer.url'] = payload.get('head_commit.committer.html_url') + config['commit_info.committer.avatar_url'] = payload.get('head_commit.committer.avatar_url') + + # Note: GitHub doesn't always return the extra information for users, so we do the lookup + # manually if possible. + if (lookup_user and not payload.get('head_commit.author.html_url') and + payload.get('head_commit.author.username')): + author_info = lookup_user(payload['head_commit.author.username']) + if author_info: + config['commit_info.author.url'] = author_info['html_url'] + config['commit_info.author.avatar_url'] = author_info['avatar_url'] + + if (lookup_user and + payload.get('head_commit.committer.username') and + not payload.get('head_commit.committer.html_url')): + committer_info = lookup_user(payload['head_commit.committer.username']) + if committer_info: + config['commit_info.committer.url'] = committer_info['html_url'] + config['commit_info.committer.avatar_url'] = committer_info['avatar_url'] + + return config.dict_value() + + +class GithubBuildTrigger(BuildTriggerHandler): + """ + BuildTrigger for GitHub that uses the archive API and buildpacks. + """ + def _get_client(self): + """ Returns an authenticated client for talking to the GitHub API. """ + return Github(self.auth_token, + base_url=github_trigger.api_endpoint(), + client_id=github_trigger.client_id(), + client_secret=github_trigger.client_secret(), + timeout=5) + + @classmethod + def service_name(cls): + return 'github' + + def is_active(self): + return 'hook_id' in self.config + + def get_repository_url(self): + source = self.config['build_source'] + return github_trigger.get_public_url(source) + + @staticmethod + def _get_error_message(ghe, default_msg): + if ghe.data.get('errors') and ghe.data['errors'][0].get('message'): + return ghe.data['errors'][0]['message'] + + return default_msg + + def activate(self, standard_webhook_url): + config = self.config + new_build_source = config['build_source'] + gh_client = self._get_client() + + # Find the GitHub repository. + try: + gh_repo = gh_client.get_repo(new_build_source) + except UnknownObjectException: + msg = 'Unable to find GitHub repository for source: %s' % new_build_source + raise TriggerActivationException(msg) + + # Add a deploy key to the GitHub repository. + public_key, private_key = generate_ssh_keypair() + config['credentials'] = [ + { + 'name': 'SSH Public Key', + 'value': public_key, + }, + ] + + try: + deploy_key = gh_repo.create_key('%s Builder' % app.config['REGISTRY_TITLE'], + public_key) + config['deploy_key_id'] = deploy_key.id + except GithubException as ghe: + default_msg = 'Unable to add deploy key to repository: %s' % new_build_source + msg = GithubBuildTrigger._get_error_message(ghe, default_msg) + raise TriggerActivationException(msg) + + # Add the webhook to the GitHub repository. + webhook_config = { + 'url': standard_webhook_url, + 'content_type': 'json', + } + + try: + hook = gh_repo.create_hook('web', webhook_config) + config['hook_id'] = hook.id + config['master_branch'] = gh_repo.default_branch + except GithubException: + default_msg = 'Unable to create webhook on repository: %s' % new_build_source + msg = GithubBuildTrigger._get_error_message(ghe, default_msg) + raise TriggerActivationException(msg) + + return config, {'private_key': private_key} + + def deactivate(self): + config = self.config + gh_client = self._get_client() + + # Find the GitHub repository. + try: + repo = gh_client.get_repo(config['build_source']) + except UnknownObjectException: + msg = 'Unable to find GitHub repository for source: %s' % config['build_source'] + raise TriggerDeactivationException(msg) + except GitHubBadCredentialsException: + msg = 'Unable to access repository to disable trigger' + raise TriggerDeactivationException(msg) + + # If the trigger uses a deploy key, remove it. + try: + if config['deploy_key_id']: + deploy_key = repo.get_key(config['deploy_key_id']) + deploy_key.delete() + except KeyError: + # There was no config['deploy_key_id'], thus this is an old trigger without a deploy key. + pass + except GithubException as ghe: + default_msg = 'Unable to remove deploy key: %s' % config['deploy_key_id'] + msg = GithubBuildTrigger._get_error_message(ghe, default_msg) + raise TriggerDeactivationException(msg) + + # Remove the webhook. + try: + hook = repo.get_hook(config['hook_id']) + hook.delete() + except GithubException as ghe: + default_msg = 'Unable to remove hook: %s' % config['hook_id'] + msg = GithubBuildTrigger._get_error_message(ghe, default_msg) + raise TriggerDeactivationException(msg) + + config.pop('hook_id', None) + self.config = config + return config + + def list_build_sources(self): + gh_client = self._get_client() + usr = gh_client.get_user() + + try: + repos = usr.get_repos() + except GithubException: + raise RepositoryReadException('Unable to list user repositories') + + namespaces = {} + has_non_personal = False + + for repository in repos: + namespace = repository.owner.login + if not namespace in namespaces: + is_personal_repo = namespace == usr.login + namespaces[namespace] = { + 'personal': is_personal_repo, + 'repos': [], + 'info': { + 'name': namespace, + 'avatar_url': repository.owner.avatar_url + } + } + + if not is_personal_repo: + has_non_personal = True + + namespaces[namespace]['repos'].append(repository.full_name) + + # In older versions of GitHub Enterprise, the get_repos call above does not + # return any non-personal repositories. In that case, we need to lookup the + # repositories manually. + # TODO: Remove this once we no longer support GHE versions <= 2.1 + if not has_non_personal: + for org in usr.get_orgs(): + repo_list = [repo.full_name for repo in org.get_repos(type='member')] + namespaces[org.name] = { + 'personal': False, + 'repos': repo_list, + 'info': { + 'name': org.name or org.login, + 'avatar_url': org.avatar_url + } + } + + entries = list(namespaces.values()) + entries.sort(key=lambda e: e['info']['name']) + return entries + + def list_build_subdirs(self): + config = self.config + gh_client = self._get_client() + source = config['build_source'] + + try: + repo = gh_client.get_repo(source) + + # Find the first matching branch. + repo_branches = self.list_field_values('branch_name') or [] + branches = find_matching_branches(config, repo_branches) + branches = branches or [repo.default_branch or 'master'] + default_commit = repo.get_branch(branches[0]).commit + commit_tree = repo.get_git_tree(default_commit.sha, recursive=True) + + return [os.path.dirname(elem.path) for elem in commit_tree.tree + if (elem.type == u'blob' and + os.path.basename(elem.path) == u'Dockerfile')] + except GithubException as ghe: + message = ghe.data.get('message', 'Unable to list contents of repository: %s' % source) + if message == 'Branch not found': + raise EmptyRepositoryException() + + raise RepositoryReadException(message) + + def load_dockerfile_contents(self): + config = self.config + gh_client = self._get_client() + + source = config['build_source'] + path = self.get_dockerfile_path() + try: + repo = gh_client.get_repo(source) + file_info = repo.get_file_contents(path) + if file_info is None: + return None + + content = file_info.content + if file_info.encoding == 'base64': + content = base64.b64decode(content) + return content + + except GithubException as ghe: + message = ghe.data.get('message', 'Unable to read Dockerfile: %s' % source) + raise RepositoryReadException(message) + + def list_field_values(self, field_name, limit=None): + if field_name == 'refs': + branches = self.list_field_values('branch_name') + tags = self.list_field_values('tag_name') + + return ([{'kind': 'branch', 'name': b} for b in branches] + + [{'kind': 'tag', 'name': tag} for tag in tags]) + + config = self.config + if field_name == 'tag_name': + try: + gh_client = self._get_client() + source = config['build_source'] + repo = gh_client.get_repo(source) + gh_tags = repo.get_tags() + if limit: + gh_tags = repo.get_tags()[0:limit] + + return [tag.name for tag in gh_tags] + except GitHubBadCredentialsException: + return [] + except GithubException: + logger.exception("Got GitHub Exception when trying to list tags for trigger %s", + self.trigger.id) + return [] + + if field_name == 'branch_name': + try: + gh_client = self._get_client() + source = config['build_source'] + repo = gh_client.get_repo(source) + gh_branches = repo.get_branches() + if limit: + gh_branches = repo.get_branches()[0:limit] + + branches = [branch.name for branch in gh_branches] + + if not repo.default_branch in branches: + branches.insert(0, repo.default_branch) + + if branches[0] != repo.default_branch: + branches.remove(repo.default_branch) + branches.insert(0, repo.default_branch) + + return branches + except GitHubBadCredentialsException: + return ['master'] + except GithubException: + logger.exception("Got GitHub Exception when trying to list branches for trigger %s", + self.trigger.id) + return ['master'] + + return None + + @classmethod + def _build_metadata_for_commit(cls, commit_sha, ref, repo): + try: + commit = repo.get_commit(commit_sha) + except GithubException: + logger.exception('Could not load commit information from GitHub') + return None + + commit_info = { + 'url': commit.html_url, + 'message': commit.commit.message, + 'date': commit.last_modified + } + + if commit.author: + commit_info['author'] = { + 'username': commit.author.login, + 'avatar_url': commit.author.avatar_url, + 'url': commit.author.html_url + } + + if commit.committer: + commit_info['committer'] = { + 'username': commit.committer.login, + 'avatar_url': commit.committer.avatar_url, + 'url': commit.committer.html_url + } + + return { + 'commit': commit_sha, + 'ref': ref, + 'default_branch': repo.default_branch, + 'git_url': repo.ssh_url, + 'commit_info': commit_info + } + + def manual_start(self, run_parameters=None): + config = self.config + source = config['build_source'] + + try: + gh_client = self._get_client() + repo = gh_client.get_repo(source) + default_branch = repo.default_branch + except GithubException as ghe: + msg = GithubBuildTrigger._get_error_message(ghe, 'Unable to start build trigger') + raise TriggerStartException(msg) + + def get_branch_sha(branch_name): + branch = repo.get_branch(branch_name) + return branch.commit.sha + + def get_tag_sha(tag_name): + tags = {tag.name: tag for tag in repo.get_tags()} + if not tag_name in tags: + raise TriggerStartException('Could not find tag in repository') + + return tags[tag_name].commit.sha + + # Find the branch or tag to build. + (commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, + default_branch) + + metadata = GithubBuildTrigger._build_metadata_for_commit(commit_sha, ref, repo) + return self.prepare_build(metadata, is_manual=True) + + def lookup_user(self, username): + try: + gh_client = self._get_client() + user = gh_client.get_user(username) + return { + 'html_url': user.html_url, + 'avatar_url': user.avatar_url + } + except GithubException: + return None + + def handle_trigger_request(self, request): + # Check the payload to see if we should skip it based on the lack of a head_commit. + payload = request.get_json() + + # This is for GitHub's probing/testing. + if 'zen' in payload: + raise ValidationRequestException() + + # Lookup the default branch for the repository. + default_branch = None + lookup_user = None + try: + repo_full_name = '%s/%s' % (payload['repository']['owner']['name'], + payload['repository']['name']) + + gh_client = self._get_client() + repo = gh_client.get_repo(repo_full_name) + default_branch = repo.default_branch + lookup_user = self.lookup_user + except GitHubBadCredentialsException: + logger.exception('Got GitHub Credentials Exception; Cannot lookup default branch') + except GithubException: + logger.exception("Got GitHub Exception when trying to start trigger %s", self.trigger.id) + raise SkipRequestException() + + logger.debug('GitHub trigger payload %s', payload) + metadata = get_transformed_webhook_payload(payload, default_branch=default_branch, + lookup_user=lookup_user) + prepared = self.prepare_build(metadata) + + # Check if we should skip this build. + raise_if_skipped_build(prepared, self.config) + return prepared diff --git a/buildtrigger/gitlabhandler.py b/buildtrigger/gitlabhandler.py new file mode 100644 index 000000000..8c1dba555 --- /dev/null +++ b/buildtrigger/gitlabhandler.py @@ -0,0 +1,432 @@ +import logging + +from functools import wraps + +from app import app + +from jsonschema import validate +from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, + TriggerDeactivationException, TriggerStartException, + SkipRequestException, InvalidPayloadException, + determine_build_ref, raise_if_skipped_build, + find_matching_branches) + +from buildtrigger.basehandler import BuildTriggerHandler + +from util.security.ssh import generate_ssh_keypair +from util.dict_wrappers import JSONPathDict, SafeDictSetter +from endpoints.api import ExternalServiceTimeout + +import gitlab +import requests + +logger = logging.getLogger(__name__) + +GITLAB_WEBHOOK_PAYLOAD_SCHEMA = { + 'type': 'object', + 'properties': { + 'ref': { + 'type': 'string', + }, + 'checkout_sha': { + 'type': 'string', + }, + 'repository': { + 'type': 'object', + 'properties': { + 'git_ssh_url': { + 'type': 'string', + }, + }, + 'required': ['git_ssh_url'], + }, + 'commits': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'url': { + 'type': 'string', + }, + 'message': { + 'type': 'string', + }, + 'timestamp': { + 'type': 'string', + }, + 'author': { + 'type': 'object', + 'properties': { + 'email': { + 'type': 'string', + }, + }, + 'required': ['email'], + }, + }, + 'required': ['url', 'message', 'timestamp'], + }, + 'minItems': 1, + } + }, + 'required': ['ref', 'checkout_sha', 'repository'], +} + +def _catch_timeouts(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except requests.exceptions.Timeout: + msg = 'Request to the GitLab API timed out' + logger.exception(msg) + raise ExternalServiceTimeout(msg) + return wrapper + + +def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None): + """ Returns the Gitlab webhook JSON payload transformed into our own payload + format. If the gl_payload is not valid, returns None. + """ + try: + validate(gl_payload, GITLAB_WEBHOOK_PAYLOAD_SCHEMA) + except Exception as exc: + raise InvalidPayloadException(exc.message) + + payload = JSONPathDict(gl_payload) + + config = SafeDictSetter() + config['commit'] = payload['checkout_sha'] + config['ref'] = payload['ref'] + config['default_branch'] = default_branch + config['git_url'] = payload['repository.git_ssh_url'] + + config['commit_info.url'] = payload['commits[0].url'] + config['commit_info.message'] = payload['commits[0].message'] + config['commit_info.date'] = payload['commits[0].timestamp'] + + # Note: Gitlab does not send full user information with the payload, so we have to + # (optionally) look it up. + author_email = payload['commits[0].author.email'] + if lookup_user and author_email: + author_info = lookup_user(author_email) + if author_info: + config['commit_info.author.username'] = author_info['username'] + config['commit_info.author.url'] = author_info['html_url'] + config['commit_info.author.avatar_url'] = author_info['avatar_url'] + + return config.dict_value() + + +class GitLabBuildTrigger(BuildTriggerHandler): + """ + BuildTrigger for GitLab. + """ + @classmethod + def service_name(cls): + return 'gitlab' + + def _get_authorized_client(self): + host = app.config.get('GITLAB_TRIGGER_CONFIG', {}).get('GITLAB_ENDPOINT', '') + auth_token = self.auth_token or 'invalid' + return gitlab.Gitlab(host, oauth_token=auth_token, timeout=5) + + def is_active(self): + return 'hook_id' in self.config + + @_catch_timeouts + def activate(self, standard_webhook_url): + config = self.config + new_build_source = config['build_source'] + gl_client = self._get_authorized_client() + + # Find the GitLab repository. + repository = gl_client.getproject(new_build_source) + if repository is False: + msg = 'Unable to find GitLab repository for source: %s' % new_build_source + raise TriggerActivationException(msg) + + # Add a deploy key to the repository. + public_key, private_key = generate_ssh_keypair() + config['credentials'] = [ + { + 'name': 'SSH Public Key', + 'value': public_key, + }, + ] + key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'], + public_key) + if key is False: + msg = 'Unable to add deploy key to repository: %s' % new_build_source + raise TriggerActivationException(msg) + config['key_id'] = key['id'] + + # Add the webhook to the GitLab repository. + hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True) + if hook is False: + msg = 'Unable to create webhook on repository: %s' % new_build_source + raise TriggerActivationException(msg) + + config['hook_id'] = hook['id'] + self.config = config + return config, {'private_key': private_key} + + def deactivate(self): + config = self.config + gl_client = self._get_authorized_client() + + # Find the GitLab repository. + repository = gl_client.getproject(config['build_source']) + if repository is False: + msg = 'Unable to find GitLab repository for source: %s' % config['build_source'] + raise TriggerDeactivationException(msg) + + # Remove the webhook. + success = gl_client.deleteprojecthook(repository['id'], config['hook_id']) + if success is False: + msg = 'Unable to remove hook: %s' % config['hook_id'] + raise TriggerDeactivationException(msg) + config.pop('hook_id', None) + + # Remove the key + success = gl_client.deletedeploykey(repository['id'], config['key_id']) + if success is False: + msg = 'Unable to remove deploy key: %s' % config['key_id'] + raise TriggerDeactivationException(msg) + config.pop('key_id', None) + + self.config = config + + return config + + @_catch_timeouts + def list_build_sources(self): + gl_client = self._get_authorized_client() + current_user = gl_client.currentuser() + if current_user is False: + raise RepositoryReadException('Unable to get current user') + + repositories = gl_client.getprojects() + if repositories is False: + raise RepositoryReadException('Unable to list user repositories') + + namespaces = {} + for repo in repositories: + owner = repo['namespace']['name'] + if not owner in namespaces: + namespaces[owner] = { + 'personal': owner == current_user['username'], + 'repos': [], + 'info': { + 'name': owner, + } + } + + namespaces[owner]['repos'].append(repo['path_with_namespace']) + + return namespaces.values() + + @_catch_timeouts + def list_build_subdirs(self): + config = self.config + gl_client = self._get_authorized_client() + new_build_source = config['build_source'] + + repository = gl_client.getproject(new_build_source) + if repository is False: + msg = 'Unable to find GitLab repository for source: %s' % new_build_source + raise RepositoryReadException(msg) + + repo_branches = gl_client.getbranches(repository['id']) + if repo_branches is False: + msg = 'Unable to find GitLab branches for source: %s' % new_build_source + raise RepositoryReadException(msg) + + branches = [branch['name'] for branch in repo_branches] + branches = find_matching_branches(config, branches) + branches = branches or [repository['default_branch'] or 'master'] + + repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0]) + if repo_tree is False: + msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source + raise RepositoryReadException(msg) + + for node in repo_tree: + if node['name'] == 'Dockerfile': + return ['/'] + + return [] + + @_catch_timeouts + def load_dockerfile_contents(self): + gl_client = self._get_authorized_client() + path = self.get_dockerfile_path() + + repository = gl_client.getproject(self.config['build_source']) + if repository is False: + return None + + branches = self.list_field_values('branch_name') + branches = find_matching_branches(self.config, branches) + if branches == []: + return None + + branch_name = branches[0] + if repository['default_branch'] in branches: + branch_name = repository['default_branch'] + + contents = gl_client.getrawfile(repository['id'], branch_name, path) + if contents is False: + return None + + return contents + + @_catch_timeouts + def list_field_values(self, field_name, limit=None): + if field_name == 'refs': + branches = self.list_field_values('branch_name') + tags = self.list_field_values('tag_name') + + return ([{'kind': 'branch', 'name': b} for b in branches] + + [{'kind': 'tag', 'name': t} for t in tags]) + + gl_client = self._get_authorized_client() + repo = gl_client.getproject(self.config['build_source']) + if repo is False: + return [] + + if field_name == 'tag_name': + tags = gl_client.getrepositorytags(repo['id']) + if tags is False: + return [] + + if limit: + tags = tags[0:limit] + + return [tag['name'] for tag in tags] + + if field_name == 'branch_name': + branches = gl_client.getbranches(repo['id']) + if branches is False: + return [] + + if limit: + branches = branches[0:limit] + + return [branch['name'] for branch in branches] + + return None + + def get_repository_url(self): + return 'https://gitlab.com/%s' % self.config['build_source'] + + @_catch_timeouts + def lookup_user(self, email): + gl_client = self._get_authorized_client() + try: + [user] = gl_client.getusers(search=email) + + return { + 'username': user['username'], + 'html_url': gl_client.host + '/' + user['username'], + 'avatar_url': user['avatar_url'] + } + except ValueError: + return None + + @_catch_timeouts + def get_metadata_for_commit(self, commit_sha, ref, repo): + gl_client = self._get_authorized_client() + commit = gl_client.getrepositorycommit(repo['id'], commit_sha) + + metadata = { + 'commit': commit['id'], + 'ref': ref, + 'default_branch': repo['default_branch'], + 'git_url': repo['ssh_url_to_repo'], + 'commit_info': { + 'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'], + 'message': commit['message'], + 'date': commit['committed_date'], + }, + } + + committer = None + if 'committer_email' in commit: + committer = self.lookup_user(commit['committer_email']) + + author = None + if 'author_email' in commit: + author = self.lookup_user(commit['author_email']) + + if committer is not None: + metadata['commit_info']['committer'] = { + 'username': committer['username'], + 'avatar_url': committer['avatar_url'], + 'url': gl_client.host + '/' + committer['username'], + } + + if author is not None: + metadata['commit_info']['author'] = { + 'username': author['username'], + 'avatar_url': author['avatar_url'], + 'url': gl_client.host + '/' + author['username'] + } + + return metadata + + @_catch_timeouts + def manual_start(self, run_parameters=None): + gl_client = self._get_authorized_client() + + repo = gl_client.getproject(self.config['build_source']) + if repo is False: + raise TriggerStartException('Could not find repository') + + def get_tag_sha(tag_name): + tags = gl_client.getrepositorytags(repo['id']) + if tags is False: + raise TriggerStartException('Could not find tags') + + for tag in tags: + if tag['name'] == tag_name: + return tag['commit']['id'] + + raise TriggerStartException('Could not find commit') + + def get_branch_sha(branch_name): + branch = gl_client.getbranch(repo['id'], branch_name) + if branch is False: + raise TriggerStartException('Could not find branch') + + return branch['commit']['id'] + + # Find the branch or tag to build. + (commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, + repo['default_branch']) + + metadata = self.get_metadata_for_commit(commit_sha, ref, repo) + return self.prepare_build(metadata, is_manual=True) + + @_catch_timeouts + def handle_trigger_request(self, request): + payload = request.get_json() + if not payload: + raise SkipRequestException() + + # Lookup the default branch. + default_branch = None + gl_client = self._get_authorized_client() + repo = gl_client.getproject(self.config['build_source']) + if repo is not False: + default_branch = repo['default_branch'] + lookup_user = self.lookup_user + + logger.debug('GitLab trigger payload %s', payload) + metadata = get_transformed_webhook_payload(payload, default_branch=default_branch, + lookup_user=lookup_user) + prepared = self.prepare_build(metadata) + + # Check if we should skip this build. + raise_if_skipped_build(prepared, self.config) + return prepared diff --git a/buildtrigger/triggerutil.py b/buildtrigger/triggerutil.py new file mode 100644 index 000000000..b60f38620 --- /dev/null +++ b/buildtrigger/triggerutil.py @@ -0,0 +1,124 @@ +import json +import io +import logging +import re + +class InvalidPayloadException(Exception): + pass + +class BuildArchiveException(Exception): + pass + +class InvalidServiceException(Exception): + pass + +class TriggerActivationException(Exception): + pass + +class TriggerDeactivationException(Exception): + pass + +class TriggerStartException(Exception): + pass + +class ValidationRequestException(Exception): + pass + +class SkipRequestException(Exception): + pass + +class EmptyRepositoryException(Exception): + pass + +class RepositoryReadException(Exception): + pass + +class TriggerProviderException(Exception): + pass + +logger = logging.getLogger(__name__) + +def determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, default_branch): + run_parameters = run_parameters or {} + + kind = '' + value = '' + + if 'refs' in run_parameters and run_parameters['refs']: + kind = run_parameters['refs']['kind'] + value = run_parameters['refs']['name'] + elif 'branch_name' in run_parameters: + kind = 'branch' + value = run_parameters['branch_name'] + + kind = kind or 'branch' + value = value or default_branch + + ref = 'refs/tags/' + value if kind == 'tag' else 'refs/heads/' + value + commit_sha = get_tag_sha(value) if kind == 'tag' else get_branch_sha(value) + return (commit_sha, ref) + + +def find_matching_branches(config, branches): + if 'branchtag_regex' in config: + try: + regex = re.compile(config['branchtag_regex']) + return [branch for branch in branches + if matches_ref('refs/heads/' + branch, regex)] + except: + pass + + return branches + + +def should_skip_commit(metadata): + if 'commit_info' in metadata: + message = metadata['commit_info']['message'] + return '[skip build]' in message or '[build skip]' in message + return False + + +def raise_if_skipped_build(prepared_build, config): + """ Raises a SkipRequestException if the given build should be skipped. """ + # Check to ensure we have metadata. + if not prepared_build.metadata: + logger.debug('Skipping request due to missing metadata for prepared build') + raise SkipRequestException() + + # Check the branchtag regex. + if 'branchtag_regex' in config: + try: + regex = re.compile(config['branchtag_regex']) + except: + regex = re.compile('.*') + + if not matches_ref(prepared_build.metadata.get('ref'), regex): + raise SkipRequestException() + + # Check the commit message. + if should_skip_commit(prepared_build.metadata): + logger.debug('Skipping request due to commit message request') + raise SkipRequestException() + + +def matches_ref(ref, regex): + match_string = ref.split('/', 1)[1] + if not regex: + return False + + m = regex.match(match_string) + if not m: + return False + + return len(m.group(0)) == len(match_string) + + +def raise_unsupported(): + raise io.UnsupportedOperation + + +def get_trigger_config(trigger): + try: + return json.loads(trigger.config) + except ValueError: + return {} diff --git a/conf/http-base.conf b/conf/http-base.conf index b7b2f01a9..737c062b5 100644 --- a/conf/http-base.conf +++ b/conf/http-base.conf @@ -5,7 +5,7 @@ real_ip_recursive on; log_format lb_pp '$remote_addr ($proxy_protocol_addr) ' '- $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent"' + '"$http_referer" "$http_user_agent"'; types_hash_max_size 2048; include /usr/local/nginx/conf/mime.types.default; diff --git a/conf/server-base.conf b/conf/server-base.conf index 185536110..2c0ba4156 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -4,6 +4,10 @@ server_name _; keepalive_timeout 5; +if ($host = "www.quay.io") { + return 301 $scheme://quay.io$request_uri; +} + if ($args ~ "_escaped_fragment_") { rewrite ^ /snapshot$uri; } diff --git a/data/buildlogs.py b/data/buildlogs.py index 17e5b397f..e9ea4a78f 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -1,5 +1,6 @@ import redis import json +import time from util.dynamic import import_class from datetime import timedelta @@ -65,7 +66,6 @@ class RedisBuildLogs(object): """ self._redis.expire(self._logs_key(build_id), ONE_DAY) - @staticmethod def _status_key(build_id): return 'builds/%s/status' % build_id @@ -88,9 +88,20 @@ class RedisBuildLogs(object): return json.loads(fetched) if fetched else None + @staticmethod + def _health_key(): + return '_health' + def check_health(self): try: - return self._redis.ping() == True + if not self._redis.ping() == True: + return False + + # Ensure we can write and read a key. + self._redis.set(self._health_key(), time.time()) + self._redis.get(self._health_key()) + + return True except redis.ConnectionError: return False diff --git a/data/database.py b/data/database.py index fb2cb2d20..dc797d9c1 100644 --- a/data/database.py +++ b/data/database.py @@ -491,12 +491,8 @@ class EmailConfirmation(BaseModel): class ImageStorage(BaseModel): uuid = CharField(default=uuid_generator, index=True, unique=True) checksum = CharField(null=True) - created = DateTimeField(null=True) - comment = TextField(null=True) - command = TextField(null=True) image_size = BigIntegerField(null=True) uncompressed_size = BigIntegerField(null=True) - aggregate_size = BigIntegerField(null=True) uploading = BooleanField(default=True, null=True) cas_path = BooleanField(default=True) diff --git a/data/migrations/versions/127905a52fdd_remove_the_deprecated_imagestorage_.py b/data/migrations/versions/127905a52fdd_remove_the_deprecated_imagestorage_.py new file mode 100644 index 000000000..06c20c015 --- /dev/null +++ b/data/migrations/versions/127905a52fdd_remove_the_deprecated_imagestorage_.py @@ -0,0 +1,32 @@ +"""Remove the deprecated imagestorage columns. + +Revision ID: 127905a52fdd +Revises: 2e0380215d01 +Create Date: 2015-09-17 15:48:56.667823 + +""" + +# revision identifiers, used by Alembic. +revision = '127905a52fdd' +down_revision = '2e0380215d01' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('imagestorage', 'comment') + op.drop_column('imagestorage', 'aggregate_size') + op.drop_column('imagestorage', 'command') + op.drop_column('imagestorage', 'created') + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('imagestorage', sa.Column('created', mysql.DATETIME(), nullable=True)) + op.add_column('imagestorage', sa.Column('command', mysql.TEXT(), nullable=True)) + op.add_column('imagestorage', sa.Column('aggregate_size', mysql.BIGINT(display_width=20), autoincrement=False, nullable=True)) + op.add_column('imagestorage', sa.Column('comment', mysql.TEXT(), nullable=True)) + ### end Alembic commands ### diff --git a/data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py b/data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py new file mode 100644 index 000000000..131c7fad0 --- /dev/null +++ b/data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py @@ -0,0 +1,24 @@ +"""Backfill image fields from image storages + +Revision ID: 2e0380215d01 +Revises: 3ff4fbc94644 +Create Date: 2015-09-15 16:57:42.850246 + +""" + +# revision identifiers, used by Alembic. +revision = '2e0380215d01' +down_revision = '3ff4fbc94644' + +from alembic import op +import sqlalchemy as sa +from util.migrate.backfill_image_fields import backfill_image_fields +from util.migrate.backfill_v1_metadata import backfill_v1_metadata + + +def upgrade(tables): + backfill_image_fields() + backfill_v1_metadata() + +def downgrade(tables): + pass diff --git a/data/model/_basequery.py b/data/model/_basequery.py index 131f860e7..1cc2d22aa 100644 --- a/data/model/_basequery.py +++ b/data/model/_basequery.py @@ -1,10 +1,22 @@ -from peewee import JOIN_LEFT_OUTER +from peewee import JOIN_LEFT_OUTER, Clause, SQL from cachetools import lru_cache from data.database import (Repository, User, Team, TeamMember, RepositoryPermission, TeamRole, Namespace, Visibility, db_for_update) +def prefix_search(field, prefix_query): + """ Returns the wildcard match for searching for the given prefix query. """ + # Escape the known wildcard characters. + prefix_query = (prefix_query + .replace('!', '!!') + .replace('%', '!%') + .replace('_', '!_') + .replace('[', '![')) + + return field ** Clause(prefix_query + '%', SQL("ESCAPE '!'")) + + def get_existing_repository(namespace_name, repository_name, for_update=False): query = (Repository .select(Repository, Namespace) @@ -25,7 +37,18 @@ def filter_to_repos_for_user(query, username=None, namespace=None, include_publi if not include_public and not username: return Repository.select().where(Repository.id == '-1') - where_clause = None + # Build a set of queries that, when unioned together, return the full set of visible repositories + # for the filters specified. + queries = [] + + where_clause = (True) + if namespace: + where_clause = (Namespace.username == namespace) + + if include_public: + queries.append(query.clone() + .where(Repository.visibility == get_public_repo_visibility(), where_clause)) + if username: UserThroughTeam = User.alias() Org = User.alias() @@ -33,37 +56,32 @@ def filter_to_repos_for_user(query, username=None, namespace=None, include_publi AdminTeamMember = TeamMember.alias() AdminUser = User.alias() - query = (query - .switch(RepositoryPermission) - .join(User, JOIN_LEFT_OUTER) - .switch(RepositoryPermission) - .join(Team, JOIN_LEFT_OUTER) - .join(TeamMember, JOIN_LEFT_OUTER) - .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == TeamMember.user)) - .switch(Repository) - .join(Org, JOIN_LEFT_OUTER, on=(Repository.namespace_user == Org.id)) - .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == AdminTeam.organization)) - .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) - .switch(AdminTeam) - .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == AdminTeamMember.team)) - .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == AdminUser.id))) + # Add repositories in which the user has permission. + queries.append(query.clone() + .switch(RepositoryPermission) + .join(User) + .where(User.username == username, where_clause)) - where_clause = ((User.username == username) | (UserThroughTeam.username == username) | - ((AdminUser.username == username) & (TeamRole.name == 'admin'))) + # Add repositories in which the user is a member of a team that has permission. + queries.append(query.clone() + .switch(RepositoryPermission) + .join(Team) + .join(TeamMember) + .join(UserThroughTeam, on=(UserThroughTeam.id == TeamMember.user)) + .where(UserThroughTeam.username == username, where_clause)) - if namespace: - where_clause = where_clause & (Namespace.username == namespace) + # Add repositories under namespaces in which the user is the org admin. + queries.append(query.clone() + .switch(Repository) + .join(Org, on=(Repository.namespace_user == Org.id)) + .join(AdminTeam, on=(Org.id == AdminTeam.organization)) + .join(TeamRole, on=(AdminTeam.role == TeamRole.id)) + .switch(AdminTeam) + .join(AdminTeamMember, on=(AdminTeam.id == AdminTeamMember.team)) + .join(AdminUser, on=(AdminTeamMember.user == AdminUser.id)) + .where(AdminUser.username == username, where_clause)) - # TODO(jschorr, jake): Figure out why the old join on Visibility was so darn slow and - # remove this hack. - if include_public: - new_clause = (Repository.visibility == get_public_repo_visibility()) - if where_clause: - where_clause = where_clause | new_clause - else: - where_clause = new_clause - - return query.where(where_clause) + return reduce(lambda l, r: l | r, queries) def get_user_organizations(username): diff --git a/data/model/image.py b/data/model/image.py index 12b6e9dd7..c8e3fb74e 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -79,11 +79,14 @@ def get_repository_images_base(namespace_name, repository_name, query_modifier): .where(Repository.name == repository_name, Namespace.username == namespace_name)) query = query_modifier(query) - return _translate_placements_to_images_with_locations(query) + return invert_placement_query_results(query) -def _translate_placements_to_images_with_locations(query): - location_list = list(query) +def invert_placement_query_results(placement_query): + """ This method will take a query which returns placements, storages, and images, and have it + return images and their storages, along with the placement set on each storage. + """ + location_list = list(placement_query) images = {} for location in location_list: @@ -192,7 +195,12 @@ def _find_or_link_image(existing_image, repo_obj, username, translations, prefer new_image = Image.create(docker_image_id=existing_image.docker_image_id, repository=repo_obj, storage=copied_storage, - ancestors=new_image_ancestry) + ancestors=new_image_ancestry, + command=existing_image.command, + created=existing_image.created, + comment=existing_image.comment, + aggregate_size=existing_image.aggregate_size) + logger.debug('Storing translation %s -> %s', existing_image.id, new_image.id) translations[existing_image.id] = new_image.id @@ -274,24 +282,15 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created # We cleanup any old checksum in case it's a retry after a fail fetched.storage.checksum = None - now = datetime.now() - # TODO stop writing to storage when all readers are removed - fetched.storage.created = now - fetched.created = now + fetched.created = datetime.now() if created_date_str is not None: try: - # TODO stop writing to storage fields when all readers are removed - parsed_created_time = dateutil.parser.parse(created_date_str).replace(tzinfo=None) - fetched.created = parsed_created_time - fetched.storage.created = parsed_created_time + fetched.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None) except: # parse raises different exceptions, so we cannot use a specific kind of handler here. pass - # TODO stop writing to storage fields when all readers are removed - fetched.storage.comment = comment - fetched.storage.command = command fetched.comment = comment fetched.command = command fetched.v1_json_metadata = v1_json_metadata @@ -304,6 +303,9 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created def set_image_size(docker_image_id, namespace_name, repository_name, image_size, uncompressed_size): + if image_size is None: + raise DataModelException('Empty image size field') + try: image = (Image .select(Image, ImageStorage) @@ -314,7 +316,6 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size, .where(Repository.name == repository_name, Namespace.username == namespace_name, Image.docker_image_id == docker_image_id) .get()) - except Image.DoesNotExist: raise DataModelException('No image with specified id and repository') @@ -326,21 +327,17 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size, try: # TODO(jschorr): Switch to this faster route once we have full ancestor aggregate_size # parent_image = Image.get(Image.id == ancestors[-1]) - # total_size = image_size + parent_image.storage.aggregate_size - total_size = (ImageStorage - .select(fn.Sum(ImageStorage.image_size)) - .join(Image) - .where(Image.id << ancestors) - .scalar()) + image_size + ancestor_size = (ImageStorage + .select(fn.Sum(ImageStorage.image_size)) + .join(Image) + .where(Image.id << ancestors) + .scalar()) - # TODO stop writing to storage when all readers are removed - image.storage.aggregate_size = total_size - image.aggregate_size = total_size + if ancestor_size is not None: + image.aggregate_size = ancestor_size + image_size except Image.DoesNotExist: pass else: - # TODO stop writing to storage when all readers are removed - image.storage.aggregate_size = image_size image.aggregate_size = image_size image.storage.save() @@ -374,24 +371,6 @@ def get_repo_image_by_storage_checksum(namespace, repository_name, storage_check raise InvalidImageException(msg) -def has_image_json(image): - """ Returns the whether there exists a JSON definition data for the image. """ - if image.v1_json_metadata: - return bool(image.v1_json_metadata) - - store = config.store - return store.exists(image.storage.locations, store.image_json_path(image.storage.uuid)) - - -def get_image_json(image): - """ Returns the JSON definition data for the image. """ - if image.v1_json_metadata: - return image.v1_json_metadata - - store = config.store - return store.get_content(image.storage.locations, store.image_json_path(image.storage.uuid)) - - def get_image_layers(image): """ Returns a list of the full layers of an image, including itself (if specified), sorted from base image outward. """ diff --git a/data/model/log.py b/data/model/log.py index ad5713d6d..21bd1bde6 100644 --- a/data/model/log.py +++ b/data/model/log.py @@ -6,6 +6,7 @@ from cachetools import lru_cache from data.database import LogEntry, LogEntryKind, User, db +# TODO: Find a way to get logs without slowing down pagination significantly. def _logs_query(selections, start_time, end_time, performer=None, repository=None, namespace=None): joined = (LogEntry .select(*selections) diff --git a/data/model/repository.py b/data/model/repository.py index eb5419560..3d7ce2db8 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -14,6 +14,10 @@ from data.database import (Repository, Namespace, RepositoryTag, Star, Image, Im logger = logging.getLogger(__name__) +def get_public_repo_visibility(): + return _basequery.get_public_repo_visibility() + + def create_repository(namespace, name, creating_user, visibility='private'): private = Visibility.get(name=visibility) namespace_user = User.get(username=namespace) @@ -64,11 +68,7 @@ def purge_repository(namespace_name, repository_name): fetched.delete_instance(recursive=True, delete_nullable=False) -def find_repository_with_garbage(filter_list=None): - # TODO(jschorr): Remove the filter once we have turned the experiment on for everyone. - if filter_list is not None and not filter_list: - return None - +def find_repository_with_garbage(): epoch_timestamp = get_epoch_timestamp() try: @@ -80,11 +80,9 @@ def find_repository_with_garbage(filter_list=None): (RepositoryTag.lifetime_end_ts <= (epoch_timestamp - Namespace.removed_tag_expiration_s))) .limit(500) + .distinct() .alias('candidates')) - if filter_list: - candidates = candidates.where(Namespace.username << filter_list) - found = (RepositoryTag .select(candidates.c.repository_id) .from_(candidates) @@ -102,11 +100,6 @@ def find_repository_with_garbage(filter_list=None): def garbage_collect_repository(namespace_name, repository_name): - # If the namespace is the async experiment, don't perform garbage collection here. - # TODO(jschorr): Remove this check once we have turned the experiment on for everyone. - if namespace_name in config.app_config.get('EXP_ASYNC_GARBAGE_COLLECTION', []): - return - repo = get_repository(namespace_name, repository_name) if repo is not None: garbage_collect_repo(repo) @@ -247,28 +240,10 @@ def get_visible_repositories(username, namespace=None, page=None, limit=None, in if not include_public and not username: return [] - fields = [Repository.name, Repository.id, Repository.description, Visibility.name, - Namespace.username] - - query = _visible_repository_query(username=username, page=page, - limit=limit, namespace=namespace, include_public=include_public, - select_models=fields) - - if limit: - query = query.limit(limit) - - if namespace: - query = query.where(Namespace.username == namespace) - - return query - - -def _visible_repository_query(username=None, include_public=True, limit=None, - page=None, namespace=None, select_models=[]): query = (Repository - .select(*select_models) # MySQL/RDS complains is there are selected models for counts. + .select(Repository.name, Repository.id, Repository.description, Namespace.username, + Repository.visibility) .distinct() - .join(Visibility) .switch(Repository) .join(Namespace, on=(Repository.namespace_user == Namespace.id)) .switch(Repository) @@ -338,36 +313,15 @@ def get_sorted_matching_repositories(prefix, only_public, checker, limit=10): # For performance reasons, we conduct the repo name and repo namespace searches on their # own. This also affords us the ability to give higher precedence to repository names matching # over namespaces, which is semantically correct. - get_search_results(Repository.name ** (prefix + '%'), with_count=True) - get_search_results(Repository.name ** (prefix + '%'), with_count=False) + get_search_results(_basequery.prefix_search(Repository.name, prefix), with_count=True) + get_search_results(_basequery.prefix_search(Repository.name, prefix), with_count=False) - get_search_results(Namespace.username ** (prefix + '%'), with_count=True) - get_search_results(Namespace.username ** (prefix + '%'), with_count=False) + get_search_results(_basequery.prefix_search(Namespace.username, prefix), with_count=True) + get_search_results(_basequery.prefix_search(Namespace.username, prefix), with_count=False) return results -def get_matching_repositories(repo_term, username=None, limit=10, include_public=True): - namespace_term = repo_term - name_term = repo_term - - visible = _visible_repository_query(username, include_public=include_public) - - search_clauses = (Repository.name ** ('%' + name_term + '%') | - Namespace.username ** ('%' + namespace_term + '%')) - - # Handle the case where the user has already entered a namespace path. - if repo_term.find('/') > 0: - parts = repo_term.split('/', 1) - namespace_term = '/'.join(parts[:-1]) - name_term = parts[-1] - - search_clauses = (Repository.name ** ('%' + name_term + '%') & - Namespace.username ** ('%' + namespace_term + '%')) - - return visible.where(search_clauses).limit(limit) - - def lookup_repository(repo_id): try: return Repository.get(Repository.id == repo_id) diff --git a/data/model/tag.py b/data/model/tag.py index 9e050c095..cddc95ebd 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -134,8 +134,7 @@ def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None): .join(Image) .where(RepositoryTag.repository == repo_obj) .where(RepositoryTag.hidden == False) - .order_by(RepositoryTag.lifetime_start_ts.desc()) - .order_by(RepositoryTag.name) + .order_by(RepositoryTag.lifetime_start_ts.desc(), RepositoryTag.name) .paginate(page, size)) if specific_tag: diff --git a/data/model/team.py b/data/model/team.py index 532d55d3a..c7d810b80 100644 --- a/data/model/team.py +++ b/data/model/team.py @@ -137,12 +137,13 @@ def add_or_invite_to_team(inviter, team, user_obj=None, email=None, requires_inv def get_matching_user_teams(team_prefix, user_obj, limit=10): + team_prefix_search = _basequery.prefix_search(Team.name, team_prefix) query = (Team .select() .join(User) .switch(Team) .join(TeamMember) - .where(TeamMember.user == user_obj, Team.name ** (team_prefix + '%')) + .where(TeamMember.user == user_obj, team_prefix_search) .distinct(Team.id) .limit(limit)) @@ -162,6 +163,7 @@ def get_organization_team(orgname, teamname): def get_matching_admined_teams(team_prefix, user_obj, limit=10): + team_prefix_search = _basequery.prefix_search(Team.name, team_prefix) admined_orgs = (_basequery.get_user_organizations(user_obj.username) .switch(Team) .join(TeamRole) @@ -172,7 +174,7 @@ def get_matching_admined_teams(team_prefix, user_obj, limit=10): .join(User) .switch(Team) .join(TeamMember) - .where(Team.name ** (team_prefix + '%'), Team.organization << (admined_orgs)) + .where(team_prefix_search, Team.organization << (admined_orgs)) .distinct(Team.id) .limit(limit)) @@ -180,8 +182,8 @@ def get_matching_admined_teams(team_prefix, user_obj, limit=10): def get_matching_teams(team_prefix, organization): - query = Team.select().where(Team.name ** (team_prefix + '%'), - Team.organization == organization) + team_prefix_search = _basequery.prefix_search(Team.name, team_prefix) + query = Team.select().where(team_prefix_search, Team.organization == organization) return query.limit(10) diff --git a/data/model/user.py b/data/model/user.py index 877ebb4c3..d05ea1693 100644 --- a/data/model/user.py +++ b/data/model/user.py @@ -203,9 +203,11 @@ def get_matching_robots(name_prefix, username, limit=10): prefix_checks = False for org in admined_orgs: - prefix_checks = prefix_checks | (User.username ** (org.username + '+' + name_prefix + '%')) + org_search = _basequery.prefix_search(User.username, org.username + '+' + name_prefix) + prefix_checks = prefix_checks | org_search - prefix_checks = prefix_checks | (User.username ** (username + '+' + name_prefix + '%')) + user_search = _basequery.prefix_search(User.username, username + '+' + name_prefix) + prefix_checks = prefix_checks | user_search return User.select().where(prefix_checks).limit(limit) @@ -493,26 +495,25 @@ def get_user_or_org_by_customer_id(customer_id): def get_matching_user_namespaces(namespace_prefix, username, limit=10): + namespace_search = _basequery.prefix_search(Namespace.username, namespace_prefix) base_query = (Namespace .select() .distinct() - .limit(limit) .join(Repository, on=(Repository.namespace_user == Namespace.id)) .join(RepositoryPermission, JOIN_LEFT_OUTER) - .where(Namespace.username ** (namespace_prefix + '%'))) + .where(namespace_search)) - return _basequery.filter_to_repos_for_user(base_query, username) + return _basequery.filter_to_repos_for_user(base_query, username).limit(limit) def get_matching_users(username_prefix, robot_namespace=None, organization=None): - direct_user_query = (User.username ** (username_prefix + '%') & - (User.organization == False) & (User.robot == False)) + user_search = _basequery.prefix_search(User.username, username_prefix) + direct_user_query = (user_search & (User.organization == False) & (User.robot == False)) if robot_namespace: robot_prefix = format_robot_username(robot_namespace, username_prefix) - direct_user_query = (direct_user_query | - (User.username ** (robot_prefix + '%') & - (User.robot == True))) + robot_search = _basequery.prefix_search(User.username, robot_prefix) + direct_user_query = (direct_user_query | (robot_search & (User.robot == True))) query = (User .select(User.username, User.email, User.robot) diff --git a/data/queue.py b/data/queue.py index b787d22be..3ce6c7a6f 100644 --- a/data/queue.py +++ b/data/queue.py @@ -67,22 +67,20 @@ class WorkQueue(object): def _item_by_id_for_update(self, queue_id): return db_for_update(QueueItem.select().where(QueueItem.id == queue_id)).get() - def get_metrics(self, require_transaction=True): - guard = self._transaction_factory(db) if require_transaction else NoopWith() - with guard: - now = datetime.utcnow() - name_match_query = self._name_match_query() + def get_metrics(self): + now = datetime.utcnow() + name_match_query = self._name_match_query() - running_query = self._running_jobs(now, name_match_query) - running_count = running_query.distinct().count() + running_query = self._running_jobs(now, name_match_query) + running_count = running_query.distinct().count() - available_query = self._available_jobs(now, name_match_query) - available_count = available_query.select(QueueItem.queue_name).distinct().count() + available_query = self._available_jobs(now, name_match_query) + available_count = available_query.select(QueueItem.queue_name).distinct().count() - available_not_running_query = self._available_jobs_not_running(now, name_match_query, - running_query) - available_not_running_count = (available_not_running_query.select(QueueItem.queue_name) - .distinct().count()) + available_not_running_query = self._available_jobs_not_running(now, name_match_query, + running_query) + available_not_running_count = (available_not_running_query.select(QueueItem.queue_name) + .distinct().count()) return (running_count, available_not_running_count, available_count) @@ -127,7 +125,10 @@ class WorkQueue(object): params['available_after'] = available_date with self._transaction_factory(db): - return str(QueueItem.create(**params).id) + r = str(QueueItem.create(**params).id) + if self._metric_queue: + self._metric_queue.put('Added', 1, dimensions={'queue': self._queue_name}) + return r def get(self, processing_time=300): """ diff --git a/dev.df b/dev.df index 683018080..13a1a4029 100644 --- a/dev.df +++ b/dev.df @@ -19,3 +19,8 @@ RUN venv/bin/pip install -r requirements.txt WORKDIR /src/quay ENV PYTHONPATH=/ ENV PATH=/venv/bin:$PATH + +RUN apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D \ + && echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" > /etc/apt/sources.list.d/docker.list \ + && apt-get update \ + && apt-get install -y docker-engine diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index d8c2a9e66..099076de2 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -49,10 +49,16 @@ class ApiException(Exception): return rv +class ExternalServiceTimeout(ApiException): + def __init__(self, error_description, payload=None): + ApiException.__init__(self, 'external_service_timeout', 520, error_description, payload) + + class InvalidRequest(ApiException): def __init__(self, error_description, payload=None): ApiException.__init__(self, 'invalid_request', 400, error_description, payload) + class InvalidResponse(ApiException): def __init__(self, error_description, payload=None): ApiException.__init__(self, 'invalid_response', 400, error_description, payload) diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 397e40ac2..c517e1f35 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -9,12 +9,12 @@ from flask import request from rfc3987 import parse as uri_parse from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue +from buildtrigger.basehandler import BuildTriggerHandler from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, ApiResource, internal_only, format_date, api, Unauthorized, NotFound, path_param, InvalidRequest, require_repo_admin) from endpoints.building import start_build, PreparedBuild -from endpoints.trigger import BuildTriggerHandler from data import database from data import model from auth.auth_context import get_authenticated_user diff --git a/endpoints/api/image.py b/endpoints/api/image.py index 3f21a8018..057e02cae 100644 --- a/endpoints/api/image.py +++ b/endpoints/api/image.py @@ -12,10 +12,7 @@ from util.cache import cache_control_flask_restful def image_view(image, image_map, include_ancestors=True): - # TODO: Remove this once we've migrated all storage data to the image records. - storage_props = image - if image.storage and image.storage.id: - storage_props = image.storage + command = image.command def docker_id(aid): if not aid or not aid in image_map: @@ -23,13 +20,12 @@ def image_view(image, image_map, include_ancestors=True): return image_map[aid].docker_image_id - command = image.command or storage_props.command image_data = { 'id': image.docker_image_id, - 'created': format_date(image.created or storage_props.created), - 'comment': image.comment or storage_props.comment, + 'created': format_date(image.created), + 'comment': image.comment, 'command': json.loads(command) if command else None, - 'size': storage_props.image_size, + 'size': image.storage.image_size, 'uploading': image.storage.uploading, 'sort_index': len(image.ancestors), } diff --git a/endpoints/api/logs.py b/endpoints/api/logs.py index d76f6d04c..92bbcee93 100644 --- a/endpoints/api/logs.py +++ b/endpoints/api/logs.py @@ -15,6 +15,7 @@ from auth import scopes from app import avatar LOGS_PER_PAGE = 50 +MAX_PAGES = 20 def log_view(log, kinds): view = { @@ -80,7 +81,7 @@ def _validate_logs_arguments(start_time, end_time, performer_name): def get_logs(start_time, end_time, performer_name=None, repository=None, namespace=None, page=None): (start_time, end_time, performer) = _validate_logs_arguments(start_time, end_time, performer_name) - page = page if page else 1 + page = min(MAX_PAGES, page if page else 1) kinds = model.log.get_log_entry_kinds() logs = model.log.list_logs(start_time, end_time, performer=performer, repository=repository, namespace=namespace, page=page, count=LOGS_PER_PAGE + 1) diff --git a/endpoints/api/repository.py b/endpoints/api/repository.py index b241a70a0..b9664864e 100644 --- a/endpoints/api/repository.py +++ b/endpoints/api/repository.py @@ -23,6 +23,7 @@ from auth.permissions import (ModifyRepositoryPermission, AdministerRepositoryPe CreateRepositoryPermission) from auth.auth_context import get_authenticated_user from auth import scopes +from util.names import REPOSITORY_NAME_REGEX logger = logging.getLogger(__name__) @@ -104,6 +105,10 @@ class RepositoryList(ApiResource): if visibility == 'private': check_allowed_private_repos(namespace_name) + # Verify that the repository name is valid. + if not REPOSITORY_NAME_REGEX.match(repository_name): + raise InvalidRequest('Invalid repository name') + repo = model.repository.create_repository(namespace_name, repository_name, owner, visibility) repo.description = req['description'] repo.save() @@ -141,6 +146,10 @@ class RepositoryList(ApiResource): starred_repos = model.repository.get_user_starred_repositories(get_authenticated_user()) star_lookup = set([repo.id for repo in starred_repos]) + # If the user asked for only public repositories, limit to only public repos. + if public and (not namespace and not starred): + username = None + # Find the matching repositories. repositories = model.repository.get_visible_repositories(username=username, limit=limit, @@ -172,6 +181,8 @@ class RepositoryList(ApiResource): def get(self, args): """ Fetch the list of repositories visible to the current user under a variety of situations. """ + if not args['namespace'] and not args['starred'] and not args['public']: + raise InvalidRequest('namespace, starred or public are required for this API call') repositories, star_lookup = self._load_repositories(args['namespace'], args['public'], args['starred'], args['limit'], @@ -192,7 +203,7 @@ class RepositoryList(ApiResource): 'namespace': repo_obj.namespace_user.username, 'name': repo_obj.name, 'description': repo_obj.description, - 'is_public': repo_obj.visibility.name == 'public' + 'is_public': repo_obj.visibility_id == model.repository.get_public_repo_visibility().id, } repo_id = repo_obj.id @@ -243,7 +254,7 @@ class Repository(RepositoryParamResource): tag_info = { 'name': tag.name, 'image_id': tag.image.docker_image_id, - 'size': tag.image.storage.aggregate_size + 'size': tag.image.aggregate_size } if tag.lifetime_start_ts > 0: diff --git a/endpoints/api/search.py b/endpoints/api/search.py index 35941ae45..ba961080f 100644 --- a/endpoints/api/search.py +++ b/endpoints/api/search.py @@ -95,38 +95,6 @@ class EntitySearch(ApiResource): } -@resource('/v1/find/repository') -class FindRepositories(ApiResource): - """ Resource for finding repositories. """ - @parse_args - @query_param('query', 'The prefix to use when querying for repositories.', type=str, default='') - @require_scope(scopes.READ_REPO) - @nickname('findRepos') - def get(self, args): - """ Get a list of repositories that match the specified prefix query. """ - prefix = args['query'] - - def repo_view(repo): - return { - 'namespace': repo.namespace_user.username, - 'name': repo.name, - 'description': repo.description - } - - username = None - user = get_authenticated_user() - if user is not None: - username = user.username - - matching = model.repository.get_matching_repositories(prefix, username) - return { - 'repositories': [repo_view(repo) for repo in matching - if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] - } - - - def search_entity_view(username, entity, get_short_name=None): kind = 'user' avatar_data = avatar.get_data_for_user(entity) diff --git a/endpoints/api/tag.py b/endpoints/api/tag.py index b8ad1906a..b657e5827 100644 --- a/endpoints/api/tag.py +++ b/endpoints/api/tag.py @@ -8,6 +8,7 @@ from endpoints.api import (resource, nickname, require_repo_read, require_repo_w from endpoints.api.image import image_view from data import model from auth.auth_context import get_authenticated_user +from util.names import TAG_ERROR, TAG_REGEX @resource('/v1/repository//tag/') @@ -85,6 +86,10 @@ class RepositoryTag(RepositoryParamResource): @validate_json_request('MoveTag') def put(self, namespace, repository, tag): """ Change which image a tag points to or create a new tag.""" + + if not TAG_REGEX.match(tag): + abort(400, TAG_ERROR) + image_id = request.get_json()['image'] image = model.image.get_repo_image(namespace, repository, image_id) if not image: @@ -100,7 +105,6 @@ class RepositoryTag(RepositoryParamResource): pass model.tag.create_or_update_tag(namespace, repository, tag, image_id) - model.repository.garbage_collect_repository(namespace, repository) username = get_authenticated_user().username log_action('move_tag' if original_image_id else 'create_tag', namespace, @@ -115,7 +119,6 @@ class RepositoryTag(RepositoryParamResource): def delete(self, namespace, repository, tag): """ Delete the specified repository tag. """ model.tag.delete_tag(namespace, repository, tag) - model.repository.garbage_collect_repository(namespace, repository) username = get_authenticated_user().username log_action('delete_tag', namespace, @@ -188,7 +191,6 @@ class RevertTag(RepositoryParamResource): # Revert the tag back to the previous image. image_id = request.get_json()['image'] model.tag.revert_tag(tag_image.repository, tag, image_id) - model.repository.garbage_collect_repository(namespace, repository) # Log the reversion. username = get_authenticated_user().username diff --git a/endpoints/api/trigger.py b/endpoints/api/trigger.py index 1582f3890..b1f37f727 100644 --- a/endpoints/api/trigger.py +++ b/endpoints/api/trigger.py @@ -8,15 +8,16 @@ from urllib import quote from urlparse import urlunparse from app import app +from buildtrigger.basehandler import BuildTriggerHandler +from buildtrigger.triggerutil import (TriggerDeactivationException, + TriggerActivationException, EmptyRepositoryException, + RepositoryReadException, TriggerStartException) from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin, log_action, request_error, query_param, parse_args, internal_only, validate_json_request, api, Unauthorized, NotFound, InvalidRequest, path_param) from endpoints.api.build import build_status_view, trigger_view, RepositoryBuildStatus from endpoints.building import start_build -from endpoints.trigger import (BuildTriggerHandler, TriggerDeactivationException, - TriggerActivationException, EmptyRepositoryException, - RepositoryReadException, TriggerStartException) from data import model from auth.permissions import (UserAdminPermission, AdministerOrganizationPermission, ReadRepositoryPermission) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 7c3094cc7..0f21273b3 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -62,16 +62,22 @@ def handle_invite_code(invite_code, user): def user_view(user): - def org_view(o): + def org_view(o, user_admin=True): admin_org = AdministerOrganizationPermission(o.username) - return { + org_response = { 'name': o.username, 'avatar': avatar.get_data_for_org(o), - 'is_org_admin': admin_org.can(), - 'can_create_repo': admin_org.can() or CreateRepositoryPermission(o.username).can(), - 'preferred_namespace': not (o.stripe_id is None) + 'can_create_repo': CreateRepositoryPermission(o.username).can(), } + if user_admin: + org_response.update({ + 'is_org_admin': admin_org.can(), + 'preferred_namespace': not (o.stripe_id is None), + }) + + return org_response + organizations = model.organization.get_user_organizations(user.username) def login_view(login): @@ -91,23 +97,29 @@ def user_view(user): user_response = { 'anonymous': False, 'username': user.username, - 'avatar': avatar.get_data_for_user(user) + 'avatar': avatar.get_data_for_user(user), } user_admin = UserAdminPermission(user.username) if user_admin.can(): user_response.update({ + 'can_create_repo': True, 'is_me': True, 'verified': user.verified, 'email': user.email, - 'organizations': [org_view(o) for o in organizations], 'logins': [login_view(login) for login in logins], - 'can_create_repo': True, 'invoice_email': user.invoice_email, 'preferred_namespace': not (user.stripe_id is None), 'tag_expiration': user.removed_tag_expiration_s, }) + user_view_perm = UserReadPermission(user.username) + if user_view_perm.can(): + user_response.update({ + 'organizations': [org_view(o, user_admin=user_admin.can()) for o in organizations], + }) + + if features.SUPER_USERS and SuperUserPermission().can(): user_response.update({ 'super_user': user and user == get_authenticated_user() and SuperUserPermission().can() diff --git a/endpoints/bitbuckettrigger.py b/endpoints/bitbuckettrigger.py index ba685450c..7045200f9 100644 --- a/endpoints/bitbuckettrigger.py +++ b/endpoints/bitbuckettrigger.py @@ -3,7 +3,8 @@ import logging from flask import request, redirect, url_for, Blueprint from flask.ext.login import current_user -from endpoints.trigger import BitbucketBuildTrigger, BuildTriggerHandler +from buildtrigger.basehandler import BuildTriggerHandler +from buildtrigger.bitbuckethandler import BitbucketBuildTrigger from endpoints.common import route_show_if from app import app from data import model diff --git a/endpoints/building.py b/endpoints/building.py index 93d76be7b..bf2c2b161 100644 --- a/endpoints/building.py +++ b/endpoints/building.py @@ -96,7 +96,7 @@ class PreparedBuild(object): def get_display_name(sha): return sha[0:7] - def tags_from_ref(self, ref, default_branch='master'): + def tags_from_ref(self, ref, default_branch=None): branch = ref.split('/')[-1] tags = {branch} diff --git a/endpoints/trigger.py b/endpoints/trigger.py deleted file mode 100644 index 5e174cdb9..000000000 --- a/endpoints/trigger.py +++ /dev/null @@ -1,1589 +0,0 @@ -import logging -import io -import os.path -import tarfile -import base64 -import re -import json - -import gitlab - -from endpoints.building import PreparedBuild -from github import (Github, UnknownObjectException, GithubException, - BadCredentialsException as GitHubBadCredentialsException) -from bitbucket import BitBucket -from tempfile import SpooledTemporaryFile -from jsonschema import validate -from data import model - -from app import app, userfiles as user_files, github_trigger, get_app_url -from util.registry.tarfileappender import TarfileAppender -from util.security.ssh import generate_ssh_keypair - - -client = app.config['HTTPCLIENT'] - - -logger = logging.getLogger(__name__) - - -TARBALL_MIME = 'application/gzip' -CHUNK_SIZE = 512 * 1024 - - -class InvalidPayloadException(Exception): - pass - -class BuildArchiveException(Exception): - pass - -class InvalidServiceException(Exception): - pass - -class TriggerActivationException(Exception): - pass - -class TriggerDeactivationException(Exception): - pass - -class TriggerStartException(Exception): - pass - -class ValidationRequestException(Exception): - pass - -class SkipRequestException(Exception): - pass - -class EmptyRepositoryException(Exception): - pass - -class RepositoryReadException(Exception): - pass - -class TriggerProviderException(Exception): - pass - - -def _determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, default_branch): - run_parameters = run_parameters or {} - - kind = '' - value = '' - - if 'refs' in run_parameters and run_parameters['refs']: - kind = run_parameters['refs']['kind'] - value = run_parameters['refs']['name'] - elif 'branch_name' in run_parameters: - kind = 'branch' - value = run_parameters['branch_name'] - - kind = kind or 'branch' - value = value or default_branch - - ref = 'refs/tags/' + value if kind == 'tag' else 'refs/heads/' + value - commit_sha = get_tag_sha(value) if kind == 'tag' else get_branch_sha(value) - return (commit_sha, ref) - - -def find_matching_branches(config, branches): - if 'branchtag_regex' in config: - try: - regex = re.compile(config['branchtag_regex']) - return [branch for branch in branches - if matches_ref('refs/heads/' + branch, regex)] - except: - pass - - return branches - -def raise_if_skipped(config, ref): - """ Raises a SkipRequestException if the given ref should be skipped. """ - if 'branchtag_regex' in config: - try: - regex = re.compile(config['branchtag_regex']) - except: - regex = re.compile('.*') - - if not matches_ref(ref, regex): - raise SkipRequestException() - -def matches_ref(ref, regex): - match_string = ref.split('/', 1)[1] - if not regex: - return False - - m = regex.match(match_string) - if not m: - return False - - return len(m.group(0)) == len(match_string) - -def should_skip_commit(message): - return '[skip build]' in message or '[build skip]' in message - -def raise_unsupported(): - raise io.UnsupportedOperation - -def get_trigger_config(trigger): - try: - return json.loads(trigger.config) - except: - return {} - - -class BuildTriggerHandler(object): - def __init__(self, trigger, override_config=None): - self.trigger = trigger - self.config = override_config or get_trigger_config(trigger) - - @property - def auth_token(self): - """ Returns the auth token for the trigger. """ - return self.trigger.auth_token - - def load_dockerfile_contents(self): - """ - Loads the Dockerfile found for the trigger's config and returns them or None if none could - be found/loaded. - """ - raise NotImplementedError - - def list_build_sources(self): - """ - Take the auth information for the specific trigger type and load the - list of build sources(repositories). - """ - raise NotImplementedError - - def list_build_subdirs(self): - """ - Take the auth information and the specified config so far and list all of - the possible subdirs containing dockerfiles. - """ - raise NotImplementedError - - def handle_trigger_request(self): - """ - Transform the incoming request data into a set of actions. Returns a PreparedBuild. - """ - raise NotImplementedError - - def is_active(self): - """ - Returns True if the current build trigger is active. Inactive means further - setup is needed. - """ - raise NotImplementedError - - def activate(self, standard_webhook_url): - """ - Activates the trigger for the service, with the given new configuration. - Returns new public and private config that should be stored if successful. - """ - raise NotImplementedError - - def deactivate(self): - """ - Deactivates the trigger for the service, removing any hooks installed in - the remote service. Returns the new config that should be stored if this - trigger is going to be re-activated. - """ - raise NotImplementedError - - def manual_start(self, run_parameters=None): - """ - Manually creates a repository build for this trigger. Returns a PreparedBuild. - """ - raise NotImplementedError - - def list_field_values(self, field_name, limit=None): - """ - Lists all values for the given custom trigger field. For example, a trigger might have a - field named "branches", and this method would return all branches. - """ - raise NotImplementedError - - def get_repository_url(self): - """ Returns the URL of the current trigger's repository. Note that this operation - can be called in a loop, so it should be as fast as possible. """ - raise NotImplementedError - - @classmethod - def service_name(cls): - """ - Particular service implemented by subclasses. - """ - raise NotImplementedError - - @classmethod - def get_handler(cls, trigger, override_config=None): - for subc in cls.__subclasses__(): - if subc.service_name() == trigger.service.name: - return subc(trigger, override_config) - - raise InvalidServiceException('Unable to find service: %s' % trigger.service.name) - - def put_config_key(self, key, value): - """ Updates a config key in the trigger, saving it to the DB. """ - self.config[key] = value - model.build.update_build_trigger(self.trigger, self.config) - - def set_auth_token(self, auth_token): - """ Sets the auth token for the trigger, saving it to the DB. """ - model.build.update_build_trigger(self.trigger, self.config, auth_token=auth_token) - - def get_dockerfile_path(self): - """ Returns the normalized path to the Dockerfile found in the subdirectory - in the config. """ - subdirectory = self.config.get('subdir', '') - if subdirectory == '/': - subdirectory = '' - else: - if not subdirectory.endswith('/'): - subdirectory = subdirectory + '/' - - return subdirectory + 'Dockerfile' - - -class BitbucketBuildTrigger(BuildTriggerHandler): - """ - BuildTrigger for Bitbucket. - """ - @classmethod - def service_name(cls): - return 'bitbucket' - - def _get_client(self): - key = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_KEY', '') - secret = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_SECRET', '') - - trigger_uuid = self.trigger.uuid - callback_url = '%s/oauth1/bitbucket/callback/trigger/%s' % (get_app_url(), trigger_uuid) - - return BitBucket(key, secret, callback_url) - - def _get_authorized_client(self): - base_client = self._get_client() - auth_token = self.auth_token or 'invalid:invalid' - token_parts = auth_token.split(':') - if len(token_parts) != 2: - token_parts = ['invalid', 'invalid'] - - (access_token, access_token_secret) = token_parts - return base_client.get_authorized_client(access_token, access_token_secret) - - def _get_repository_client(self): - source = self.config['build_source'] - (namespace, name) = source.split('/') - bitbucket_client = self._get_authorized_client() - return bitbucket_client.for_namespace(namespace).repositories().get(name) - - def _get_default_branch(self, repository, default_value='master'): - (result, data, _) = repository.get_main_branch() - if result: - return data['name'] - - return default_value - - def get_oauth_url(self): - bitbucket_client = self._get_client() - (result, data, err_msg) = bitbucket_client.get_authorization_url() - if not result: - raise RepositoryReadException(err_msg) - - return data - - def exchange_verifier(self, verifier): - bitbucket_client = self._get_client() - access_token = self.config.get('access_token', '') - access_token_secret = self.auth_token - - # Exchange the verifier for a new access token. - (result, data, _) = bitbucket_client.verify_token(access_token, access_token_secret, verifier) - if not result: - return False - - # Save the updated access token and secret. - self.set_auth_token(data[0] + ':' + data[1]) - - # Retrieve the current authorized user's information and store the username in the config. - authorized_client = self._get_authorized_client() - (result, data, _) = authorized_client.get_current_user() - if not result: - return False - - username = data['user']['username'] - self.put_config_key('username', username) - return True - - def is_active(self): - return 'webhook_id' in self.config - - def activate(self, standard_webhook_url): - config = self.config - - # Add a deploy key to the repository. - public_key, private_key = generate_ssh_keypair() - config['credentials'] = [ - { - 'name': 'SSH Public Key', - 'value': public_key, - }, - ] - - repository = self._get_repository_client() - (result, created_deploykey, err_msg) = repository.deploykeys().create( - app.config['REGISTRY_TITLE'] + ' webhook key', public_key) - - if not result: - msg = 'Unable to add deploy key to repository: %s' % err_msg - raise TriggerActivationException(msg) - - config['deploy_key_id'] = created_deploykey['pk'] - - # Add a webhook callback. - description = 'Webhook for invoking builds on %s' % app.config['REGISTRY_TITLE_SHORT'] - webhook_events = ['repo:push'] - (result, created_webhook, err_msg) = repository.webhooks().create( - description, standard_webhook_url, webhook_events) - - if not result: - msg = 'Unable to add webhook to repository: %s' % err_msg - raise TriggerActivationException(msg) - - config['webhook_id'] = created_webhook['uuid'] - self.config = config - return config, {'private_key': private_key} - - def deactivate(self): - config = self.config - - webhook_id = config.pop('webhook_id', None) - deploy_key_id = config.pop('deploy_key_id', None) - repository = self._get_repository_client() - - # Remove the webhook. - if webhook_id is not None: - (result, _, err_msg) = repository.webhooks().delete(webhook_id) - if not result: - msg = 'Unable to remove webhook from repository: %s' % err_msg - raise TriggerDeactivationException(msg) - - # Remove the public key. - if deploy_key_id is not None: - (result, _, err_msg) = repository.deploykeys().delete(deploy_key_id) - if not result: - msg = 'Unable to remove deploy key from repository: %s' % err_msg - raise TriggerDeactivationException(msg) - - return config - - def list_build_sources(self): - bitbucket_client = self._get_authorized_client() - (result, data, err_msg) = bitbucket_client.get_visible_repositories() - if not result: - raise RepositoryReadException('Could not read repository list: ' + err_msg) - - namespaces = {} - for repo in data: - if not repo['scm'] == 'git': - continue - - owner = repo['owner'] - if not owner in namespaces: - namespaces[owner] = { - 'personal': owner == self.config.get('username'), - 'repos': [], - 'info': { - 'name': owner - } - } - - namespaces[owner]['repos'].append(owner + '/' + repo['slug']) - - return namespaces.values() - - def list_build_subdirs(self): - config = self.config - repository = self._get_repository_client() - - # Find the first matching branch. - repo_branches = self.list_field_values('branch_name') or [] - branches = find_matching_branches(config, repo_branches) - if not branches: - branches = [self._get_default_branch(repository)] - - (result, data, err_msg) = repository.get_path_contents('', revision=branches[0]) - if not result: - raise RepositoryReadException(err_msg) - - files = set([f['path'] for f in data['files']]) - if 'Dockerfile' in files: - return ['/'] - - return [] - - def load_dockerfile_contents(self): - repository = self._get_repository_client() - path = self.get_dockerfile_path() - - (result, data, err_msg) = repository.get_raw_path_contents(path, revision='master') - if not result: - raise RepositoryReadException(err_msg) - - return data - - def list_field_values(self, field_name, limit=None): - source = self.config['build_source'] - (namespace, name) = source.split('/') - - bitbucket_client = self._get_authorized_client() - repository = bitbucket_client.for_namespace(namespace).repositories().get(name) - - if field_name == 'refs': - (result, data, _) = repository.get_branches_and_tags() - if not result: - return None - - branches = [b['name'] for b in data['branches']] - tags = [t['name'] for t in data['tags']] - - return ([{'kind': 'branch', 'name': b} for b in branches] + - [{'kind': 'tag', 'name': tag} for tag in tags]) - - if field_name == 'tag_name': - (result, data, _) = repository.get_tags() - if not result: - return None - - tags = list(data.keys()) - if limit: - tags = tags[0:limit] - - return tags - - if field_name == 'branch_name': - (result, data, _) = repository.get_branches() - if not result: - return None - - branches = list(data.keys()) - if limit: - branches = branches[0:limit] - - return branches - - return None - - _BITBUCKET_COMMIT_URL = 'https://bitbucket.org/%s/%s/commits/%s' - - def _prepare_build(self, commit_sha, ref, is_manual, target=None, actor=None): - def _build_user_block(info): - return { - 'username': info['username'], - 'url': info['links']['html']['href'], - 'avatar_url': info['links']['avatar']['href'], - 'display_name': info['display_name'] - } - - config = self.config - repository = self._get_repository_client() - - # Lookup the default branch associated with the repository. We use this when building - # the tags. - default_branch = self._get_default_branch(repository) - - # Lookup the commit sha (if necessary) - data = {} - if target is None: - (result, data, _) = repository.changesets().get(commit_sha) - if not result: - raise TriggerStartException('Could not lookup commit SHA') - - namespace = repository.namespace - name = repository.repository_name - - # Build the commit information. - commit_url = self._BITBUCKET_COMMIT_URL % (namespace, name, commit_sha) - if target is not None and 'links' in target: - commit_url = target['links']['html']['href'] - - commit_info = { - 'url': commit_url, - 'message': target['message'] if target else data['message'], - 'date': target['date'] if target else data['timestamp'] - } - - # Add the commit's author. - if target is not None and target.get('author') and 'user' in target['author']: - commit_info['author'] = _build_user_block(target['author']['user']) - elif data.get('raw_author'): - # Try to lookup the author by email address. The raw_author field (if it exists) is returned - # in the form: "Joseph Schorr " - match = re.compile(r'.*<(.+)>').match(data['raw_author']) - if match: - email_address = match.group(1) - bitbucket_client = self._get_authorized_client() - (result, data, _) = bitbucket_client.accounts().get_profile(email_address) - if result: - commit_info['author'] = { - 'username': data['user']['username'], - 'url': 'https://bitbucket.org/%s/' % data['user']['username'], - 'avatar_url': data['user']['avatar'] - } - - # Add the commit's actor (committer). - if actor is not None: - commit_info['committer'] = _build_user_block(actor) - - metadata = { - 'commit': commit_sha, - 'ref': ref, - 'default_branch': default_branch, - 'git_url': 'git@bitbucket.org:%s/%s.git' % (namespace, name), - 'commit_info': commit_info - } - - prepared = PreparedBuild(self.trigger) - prepared.tags_from_ref(ref, default_branch) - prepared.name_from_sha(commit_sha) - prepared.subdirectory = config['subdir'] - prepared.metadata = metadata - prepared.is_manual = is_manual - - return prepared - - def handle_trigger_request(self, request): - payload = request.get_json() - if not payload or not 'push' in payload: - logger.debug('Skipping BitBucket request due to missing push data in payload') - raise SkipRequestException() - - push_payload = payload['push'] - if not 'changes' in push_payload or not push_payload['changes']: - logger.debug('Skipping BitBucket request due to empty changes list') - raise SkipRequestException() - - # Make sure we have a new change. - changes = push_payload['changes'] - last_change = changes[-1] - if not last_change.get('new'): - logger.debug('Skipping BitBucket request due to change being a deletion') - raise SkipRequestException() - - change_info = last_change['new'] - change_target = change_info.get('target') - if not change_target: - logger.debug('Skipping BitBucket request due to missing change target') - raise SkipRequestException() - - # Check if this build should be skipped by commit message. - commit_message = change_target.get('message', '') - if should_skip_commit(commit_message): - logger.debug('Skipping BitBucket request due to commit message request') - raise SkipRequestException() - - # Check to see if this build should be skipped by ref. - ref = ('refs/heads/' + change_info['name'] if change_info['type'] == 'branch' - else 'refs/tags/' + change_info['name']) - - logger.debug('Checking BitBucket request: %s', ref) - raise_if_skipped(self.config, ref) - - # Prepare the build. - commit_sha = change_target['hash'] - return self._prepare_build(commit_sha, ref, False, target=change_target, - actor=payload.get('actor')) - - - def manual_start(self, run_parameters=None): - run_parameters = run_parameters or {} - repository = self._get_repository_client() - - def get_branch_sha(branch_name): - # Lookup the commit SHA for the branch. - (result, data, _) = repository.get_branches() - if not result or not branch_name in data: - raise TriggerStartException('Could not find branch commit SHA') - - return data[branch_name]['node'] - - def get_tag_sha(tag_name): - # Lookup the commit SHA for the tag. - (result, data, _) = repository.get_tags() - if not result or not tag_name in data: - raise TriggerStartException('Could not find tag commit SHA') - - return data[tag_name]['node'] - - # Find the branch or tag to build. - (commit_sha, ref) = _determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, - self._get_default_branch(repository)) - - return self._prepare_build(commit_sha, ref, True) - - def get_repository_url(self): - source = self.config['build_source'] - (namespace, name) = source.split('/') - return 'https://bitbucket.org/%s/%s' % (namespace, name) - - -class GithubBuildTrigger(BuildTriggerHandler): - """ - BuildTrigger for GitHub that uses the archive API and buildpacks. - """ - def _get_client(self): - return Github(self.auth_token, - base_url=github_trigger.api_endpoint(), - client_id=github_trigger.client_id(), - client_secret=github_trigger.client_secret()) - - @classmethod - def service_name(cls): - return 'github' - - def is_active(self): - return 'hook_id' in self.config - - def activate(self, standard_webhook_url): - config = self.config - new_build_source = config['build_source'] - gh_client = self._get_client() - - # Find the GitHub repository. - try: - gh_repo = gh_client.get_repo(new_build_source) - except UnknownObjectException: - msg = 'Unable to find GitHub repository for source: %s' % new_build_source - raise TriggerActivationException(msg) - - # Add a deploy key to the GitHub repository. - public_key, private_key = generate_ssh_keypair() - config['credentials'] = [ - { - 'name': 'SSH Public Key', - 'value': public_key, - }, - ] - try: - deploy_key = gh_repo.create_key('%s Builder' % app.config['REGISTRY_TITLE'], - public_key) - config['deploy_key_id'] = deploy_key.id - except GithubException: - msg = 'Unable to add deploy key to repository: %s' % new_build_source - raise TriggerActivationException(msg) - - # Add the webhook to the GitHub repository. - webhook_config = { - 'url': standard_webhook_url, - 'content_type': 'json', - } - try: - hook = gh_repo.create_hook('web', webhook_config) - config['hook_id'] = hook.id - config['master_branch'] = gh_repo.default_branch - except GithubException: - msg = 'Unable to create webhook on repository: %s' % new_build_source - raise TriggerActivationException(msg) - - return config, {'private_key': private_key} - - def deactivate(self): - config = self.config - gh_client = self._get_client() - - # Find the GitHub repository. - try: - repo = gh_client.get_repo(config['build_source']) - except UnknownObjectException: - msg = 'Unable to find GitHub repository for source: %s' % config['build_source'] - raise TriggerDeactivationException(msg) - except GitHubBadCredentialsException: - msg = 'Unable to access repository to disable trigger' - raise TriggerDeactivationException(msg) - - # If the trigger uses a deploy key, remove it. - try: - if config['deploy_key_id']: - deploy_key = repo.get_key(config['deploy_key_id']) - deploy_key.delete() - except KeyError: - # There was no config['deploy_key_id'], thus this is an old trigger without a deploy key. - pass - except GithubException: - msg = 'Unable to remove deploy key: %s' % config['deploy_key_id'] - raise TriggerDeactivationException(msg) - - # Remove the webhook. - try: - hook = repo.get_hook(config['hook_id']) - hook.delete() - except GithubException: - msg = 'Unable to remove hook: %s' % config['hook_id'] - raise TriggerDeactivationException(msg) - - config.pop('hook_id', None) - self.config = config - return config - - def list_build_sources(self): - gh_client = self._get_client() - usr = gh_client.get_user() - - try: - repos = usr.get_repos() - except GithubException: - raise RepositoryReadException('Unable to list user repositories') - - namespaces = {} - has_non_personal = False - - for repository in repos: - namespace = repository.owner.login - if not namespace in namespaces: - is_personal_repo = namespace == usr.login - namespaces[namespace] = { - 'personal': is_personal_repo, - 'repos': [], - 'info': { - 'name': namespace, - 'avatar_url': repository.owner.avatar_url - } - } - - if not is_personal_repo: - has_non_personal = True - - namespaces[namespace]['repos'].append(repository.full_name) - - # In older versions of GitHub Enterprise, the get_repos call above does not - # return any non-personal repositories. In that case, we need to lookup the - # repositories manually. - # TODO: Remove this once we no longer support GHE versions <= 2.1 - if not has_non_personal: - for org in usr.get_orgs(): - repo_list = [repo.full_name for repo in org.get_repos(type='member')] - namespaces[org.name] = { - 'personal': False, - 'repos': repo_list, - 'info': { - 'name': org.name or org.login, - 'avatar_url': org.avatar_url - } - } - - entries = list(namespaces.values()) - entries.sort(key=lambda e: e['info']['name']) - return entries - - def list_build_subdirs(self): - config = self.config - gh_client = self._get_client() - source = config['build_source'] - - try: - repo = gh_client.get_repo(source) - - # Find the first matching branch. - repo_branches = self.list_field_values('branch_name') or [] - branches = find_matching_branches(config, repo_branches) - branches = branches or [repo.default_branch or 'master'] - default_commit = repo.get_branch(branches[0]).commit - commit_tree = repo.get_git_tree(default_commit.sha, recursive=True) - - return [os.path.dirname(elem.path) for elem in commit_tree.tree - if (elem.type == u'blob' and - os.path.basename(elem.path) == u'Dockerfile')] - except GithubException as ge: - message = ge.data.get('message', 'Unable to list contents of repository: %s' % source) - if message == 'Branch not found': - raise EmptyRepositoryException() - - raise RepositoryReadException(message) - - - def load_dockerfile_contents(self): - config = self.config - gh_client = self._get_client() - - source = config['build_source'] - path = self.get_dockerfile_path() - try: - repo = gh_client.get_repo(source) - file_info = repo.get_file_contents(path) - if file_info is None: - return None - - content = file_info.content - if file_info.encoding == 'base64': - content = base64.b64decode(content) - return content - - except GithubException as ge: - message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source) - raise RepositoryReadException(message) - - @staticmethod - def _build_commit_info(repo, payload, commit_sha): - if repo: - return GithubBuildTrigger._build_repo_commit_info(repo, commit_sha) - else: - return GithubBuildTrigger._build_payload_commit_info(payload, commit_sha) - - @staticmethod - def _build_payload_commit_info(payload, commit_sha): - head_commit = payload.get('head_commit', {}) - sender = payload.get('sender', {}) - - commit_info = { - 'url': head_commit.get('url', ''), - 'message': head_commit.get('message', ''), - 'date': head_commit.get('timestamp', ''), - } - - if 'author' in head_commit: - commit_info['author'] = { - 'username': head_commit['author'].get('username'), - } - - if head_commit['author']['username'] == sender.get('login'): - commit_info['author']['avatar_url'] = sender.get('avatar_url', '') - commit_info['author']['url'] = sender.get('html_url', '') - - if 'committer' in head_commit: - commit_info['committer'] = { - 'username': head_commit['committer'].get('username'), - } - - if head_commit['committer']['username'] == sender.get('login'): - commit_info['committer']['avatar_url'] = sender.get('avatar_url', '') - commit_info['committer']['url'] = sender.get('html_url', '') - - return commit_info - - @staticmethod - def _build_repo_commit_info(repo, commit_sha): - try: - commit = repo.get_commit(commit_sha) - except GithubException: - logger.exception('Could not load data for commit') - return - - commit_info = { - 'url': commit.html_url, - 'message': commit.commit.message, - 'date': commit.last_modified - } - - if commit.author: - commit_info['author'] = { - 'username': commit.author.login, - 'avatar_url': commit.author.avatar_url, - 'url': commit.author.html_url - } - - if commit.committer: - commit_info['committer'] = { - 'username': commit.committer.login, - 'avatar_url': commit.committer.avatar_url, - 'url': commit.committer.html_url - } - - return commit_info - - @staticmethod - def _prepare_tarball(repo, commit_sha): - # Prepare the download and upload URLs - archive_link = repo.get_archive_link('tarball', commit_sha) - download_archive = client.get(archive_link, stream=True) - tarball_subdir = '' - - with SpooledTemporaryFile(CHUNK_SIZE) as tarball: - for chunk in download_archive.iter_content(CHUNK_SIZE): - tarball.write(chunk) - - # Seek to position 0 to make tarfile happy - tarball.seek(0) - - # Pull out the name of the subdir that GitHub generated - with tarfile.open(fileobj=tarball) as archive: - tarball_subdir = archive.getnames()[0] - - # Seek to position 0 to make tarfile happy. - tarball.seek(0) - - entries = { - tarball_subdir + '/.git/HEAD': commit_sha, - tarball_subdir + '/.git/objects/': None, - tarball_subdir + '/.git/refs/': None - } - - appender = TarfileAppender(tarball, entries).get_stream() - dockerfile_id = user_files.store_file(appender, TARBALL_MIME) - - logger.debug('Successfully prepared job') - - return tarball_subdir, dockerfile_id - - - def _get_payload(self, payload, *args): - current = payload - for arg in args: - current = current.get(arg, {}) - - return current - - - def _prepare_build(self, ref, commit_sha, is_manual, repo=None, payload=None): - config = self.config - prepared = PreparedBuild(self.trigger) - - # If the trigger isn't using git, prepare the buildpack. - if self.trigger.private_key is None: - if repo is None: - raise SkipRequestException() - - tarball_subdir, dockerfile_id = GithubBuildTrigger._prepare_tarball(repo, commit_sha) - - prepared.subdirectory = os.path.join(tarball_subdir, config['subdir']) - prepared.dockerfile_id = dockerfile_id - else: - prepared.subdirectory = config['subdir'] - - # Set the name. - prepared.name_from_sha(commit_sha) - - # Set the tag(s). - if repo: - default_branch = repo.default_branch - else: - default_branch = self._get_payload(payload, 'repository', 'default_branch') - - prepared.tags_from_ref(ref, default_branch) - - # Build and set the metadata. - metadata = { - 'commit': commit_sha, - 'ref': ref, - 'default_branch': default_branch, - 'git_url': repo.ssh_url if repo else self._get_payload(payload, 'repository', 'ssh_url'), - } - - # add the commit info. - commit_info = GithubBuildTrigger._build_commit_info(repo, payload, commit_sha) - if commit_info is not None: - metadata['commit_info'] = commit_info - - prepared.metadata = metadata - prepared.is_manual = is_manual - return prepared - - - def handle_trigger_request(self, request): - # Check the payload to see if we should skip it based on the lack of a head_commit. - payload = request.get_json() - if not payload or payload.get('head_commit') is None: - raise SkipRequestException() - - # This is for GitHub's probing/testing. - if 'zen' in payload: - raise ValidationRequestException() - - logger.debug('GitHub trigger payload %s', payload) - - ref = payload['ref'] - commit_sha = payload['head_commit']['id'] - commit_message = payload['head_commit'].get('message', '') - - # Check if this build should be skipped by commit message. - if should_skip_commit(commit_message): - raise SkipRequestException() - - # Check to see if this build should be skipped by ref. - raise_if_skipped(self.config, ref) - - try: - repo_full_name = '%s/%s' % (payload['repository']['owner']['name'], - payload['repository']['name']) - - gh_client = self._get_client() - repo = gh_client.get_repo(repo_full_name) - return self._prepare_build(ref, commit_sha, False, repo=repo) - except GitHubBadCredentialsException: - logger.exception('Got GitHub Credentials Exception, retrying with a manual payload') - return self._prepare_build(ref, commit_sha, False, payload=payload) - except GithubException: - logger.exception("Got GitHub Exception when trying to start trigger %s", self.trigger.id) - raise SkipRequestException() - - - def manual_start(self, run_parameters=None): - config = self.config - source = config['build_source'] - - try: - gh_client = self._get_client() - repo = gh_client.get_repo(source) - default_branch = repo.default_branch - except GithubException as ghe: - raise TriggerStartException(ghe.data['message']) - - def get_branch_sha(branch_name): - branch = repo.get_branch(branch_name) - return branch.commit.sha - - def get_tag_sha(tag_name): - tags = {tag.name: tag for tag in repo.get_tags()} - if not tag_name in tags: - raise TriggerStartException('Could not find tag in repository') - - return tags[tag_name].commit.sha - - # Find the branch or tag to build. - (commit_sha, ref) = _determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, - default_branch) - - return self._prepare_build(ref, commit_sha, True, repo=repo) - - - def list_field_values(self, field_name, limit=None): - if field_name == 'refs': - branches = self.list_field_values('branch_name') - tags = self.list_field_values('tag_name') - - return ([{'kind': 'branch', 'name': b} for b in branches] + - [{'kind': 'tag', 'name': tag} for tag in tags]) - - config = self.config - if field_name == 'tag_name': - try: - gh_client = self._get_client() - source = config['build_source'] - repo = gh_client.get_repo(source) - gh_tags = repo.get_tags() - if limit: - gh_tags = repo.get_tags()[0:limit] - - return [tag.name for tag in gh_tags] - except GitHubBadCredentialsException: - return [] - except GithubException: - logger.exception("Got GitHub Exception when trying to list tags for trigger %s", - self.trigger.id) - return [] - - if field_name == 'branch_name': - try: - gh_client = self._get_client() - source = config['build_source'] - repo = gh_client.get_repo(source) - gh_branches = repo.get_branches() - if limit: - gh_branches = repo.get_branches()[0:limit] - - branches = [branch.name for branch in gh_branches] - - if not repo.default_branch in branches: - branches.insert(0, repo.default_branch) - - if branches[0] != repo.default_branch: - branches.remove(repo.default_branch) - branches.insert(0, repo.default_branch) - - return branches - except GitHubBadCredentialsException: - return ['master'] - except GithubException: - logger.exception("Got GitHub Exception when trying to list branches for trigger %s", - self.trigger.id) - return ['master'] - - return None - - def get_repository_url(self): - from app import github_trigger - source = self.config['build_source'] - return github_trigger.get_public_url(source) - - -class CustomBuildTrigger(BuildTriggerHandler): - payload_schema = { - 'type': 'object', - 'properties': { - 'commit': { - 'type': 'string', - 'description': 'first 7 characters of the SHA-1 identifier for a git commit', - 'pattern': '^([A-Fa-f0-9]{7,})$', - }, - 'ref': { - 'type': 'string', - 'description': 'git reference for a git commit', - 'pattern': '^refs\/(heads|tags|remotes)\/(.+)$', - }, - 'default_branch': { - 'type': 'string', - 'description': 'default branch of the git repository', - }, - 'commit_info': { - 'type': 'object', - 'description': 'metadata about a git commit', - 'properties': { - 'url': { - 'type': 'string', - 'description': 'URL to view a git commit', - }, - 'message': { - 'type': 'string', - 'description': 'git commit message', - }, - 'date': { - 'type': 'string', - 'description': 'timestamp for a git commit' - }, - 'author': { - 'type': 'object', - 'description': 'metadata about the author of a git commit', - 'properties': { - 'username': { - 'type': 'string', - 'description': 'username of the author', - }, - 'url': { - 'type': 'string', - 'description': 'URL to view the profile of the author', - }, - 'avatar_url': { - 'type': 'string', - 'description': 'URL to view the avatar of the author', - }, - }, - 'required': ['username', 'url', 'avatar_url'], - }, - 'committer': { - 'type': 'object', - 'description': 'metadata about the committer of a git commit', - 'properties': { - 'username': { - 'type': 'string', - 'description': 'username of the committer', - }, - 'url': { - 'type': 'string', - 'description': 'URL to view the profile of the committer', - }, - 'avatar_url': { - 'type': 'string', - 'description': 'URL to view the avatar of the committer', - }, - }, - 'required': ['username', 'url', 'avatar_url'], - }, - }, - 'required': ['url', 'message', 'date'], - }, - }, - 'required': ['commit', 'ref', 'default_branch'], - } - - @classmethod - def service_name(cls): - return 'custom-git' - - def is_active(self): - return self.config.has_key('credentials') - - def _metadata_from_payload(self, payload): - try: - metadata = json.loads(payload) - validate(metadata, self.payload_schema) - except Exception as e: - raise InvalidPayloadException(e.message) - return metadata - - def handle_trigger_request(self, request): - # Skip if there is no payload. - payload = request.data - if not payload: - raise InvalidPayloadException() - - logger.debug('Payload %s', payload) - - # Skip if the commit message matches. - metadata = self._metadata_from_payload(payload) - if should_skip_commit(metadata.get('commit_info', {}).get('message', '')): - raise SkipRequestException() - - # The build source is the canonical git URL used to clone. - config = self.config - metadata['git_url'] = config['build_source'] - - prepared = PreparedBuild(self.trigger) - prepared.tags_from_ref(metadata['ref']) - prepared.name_from_sha(metadata['commit']) - prepared.subdirectory = config['subdir'] - prepared.metadata = metadata - prepared.is_manual = False - - return prepared - - def manual_start(self, run_parameters=None): - # commit_sha is the only required parameter - commit_sha = run_parameters.get('commit_sha') - if commit_sha is None: - raise TriggerStartException('missing required parameter') - - config = self.config - metadata = { - 'commit': commit_sha, - 'git_url': config['build_source'], - } - - prepared = PreparedBuild(self.trigger) - prepared.tags = [commit_sha[:7]] - prepared.name_from_sha(commit_sha) - prepared.subdirectory = config['subdir'] - prepared.metadata = metadata - prepared.is_manual = True - - return prepared - - def activate(self, standard_webhook_url): - config = self.config - public_key, private_key = generate_ssh_keypair() - config['credentials'] = [ - { - 'name': 'SSH Public Key', - 'value': public_key, - }, - { - 'name': 'Webhook Endpoint URL', - 'value': standard_webhook_url, - }, - ] - self.config = config - return config, {'private_key': private_key} - - def deactivate(self): - config = self.config - config.pop('credentials', None) - self.config = config - return config - - def get_repository_url(self): - return None - - -class GitLabBuildTrigger(BuildTriggerHandler): - """ - BuildTrigger for GitLab. - """ - @classmethod - def service_name(cls): - return 'gitlab' - - def _get_authorized_client(self): - host = app.config.get('GITLAB_TRIGGER_CONFIG', {}).get('GITLAB_ENDPOINT', '') - auth_token = self.auth_token or 'invalid' - return gitlab.Gitlab(host, oauth_token=auth_token) - - def is_active(self): - return 'hook_id' in self.config - - def activate(self, standard_webhook_url): - config = self.config - new_build_source = config['build_source'] - gl_client = self._get_authorized_client() - - # Find the GitLab repository. - repository = gl_client.getproject(new_build_source) - if repository is False: - msg = 'Unable to find GitLab repository for source: %s' % new_build_source - raise TriggerActivationException(msg) - - # Add a deploy key to the repository. - public_key, private_key = generate_ssh_keypair() - config['credentials'] = [ - { - 'name': 'SSH Public Key', - 'value': public_key, - }, - ] - key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'], - public_key) - if key is False: - msg = 'Unable to add deploy key to repository: %s' % new_build_source - raise TriggerActivationException(msg) - config['key_id'] = key['id'] - - # Add the webhook to the GitLab repository. - hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True) - if hook is False: - msg = 'Unable to create webhook on repository: %s' % new_build_source - raise TriggerActivationException(msg) - - config['hook_id'] = hook['id'] - self.config = config - return config, {'private_key': private_key} - - def deactivate(self): - config = self.config - gl_client = self._get_authorized_client() - - # Find the GitLab repository. - repository = gl_client.getproject(config['build_source']) - if repository is False: - msg = 'Unable to find GitLab repository for source: %s' % config['build_source'] - raise TriggerDeactivationException(msg) - - # Remove the webhook. - success = gl_client.deleteprojecthook(repository['id'], config['hook_id']) - if success is False: - msg = 'Unable to remove hook: %s' % config['hook_id'] - raise TriggerDeactivationException(msg) - config.pop('hook_id', None) - - # Remove the key - success = gl_client.deletedeploykey(repository['id'], config['key_id']) - if success is False: - msg = 'Unable to remove deploy key: %s' % config['key_id'] - raise TriggerDeactivationException(msg) - config.pop('key_id', None) - - self.config = config - - return config - - def list_build_sources(self): - gl_client = self._get_authorized_client() - current_user = gl_client.currentuser() - if current_user is False: - raise RepositoryReadException('Unable to get current user') - - repositories = gl_client.getprojects() - if repositories is False: - raise RepositoryReadException('Unable to list user repositories') - - namespaces = {} - for repo in repositories: - owner = repo['namespace']['name'] - if not owner in namespaces: - namespaces[owner] = { - 'personal': owner == current_user['username'], - 'repos': [], - 'info': { - 'name': owner, - } - } - - namespaces[owner]['repos'].append(repo['path_with_namespace']) - - return namespaces.values() - - def list_build_subdirs(self): - config = self.config - gl_client = self._get_authorized_client() - new_build_source = config['build_source'] - - repository = gl_client.getproject(new_build_source) - if repository is False: - msg = 'Unable to find GitLab repository for source: %s' % new_build_source - raise RepositoryReadException(msg) - - repo_branches = gl_client.getbranches(repository['id']) - if repo_branches is False: - msg = 'Unable to find GitLab branches for source: %s' % new_build_source - raise RepositoryReadException(msg) - - branches = [branch['name'] for branch in repo_branches] - branches = find_matching_branches(config, branches) - branches = branches or [repository['default_branch'] or 'master'] - - repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0]) - if repo_tree is False: - msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source - raise RepositoryReadException(msg) - - for node in repo_tree: - if node['name'] == 'Dockerfile': - return ['/'] - - return [] - - def load_dockerfile_contents(self): - gl_client = self._get_authorized_client() - path = self.get_dockerfile_path() - - repository = gl_client.getproject(self.config['build_source']) - if repository is False: - return None - - branches = self.list_field_values('branch_name') - branches = find_matching_branches(self.config, branches) - if branches == []: - return None - - branch_name = branches[0] - if repository['default_branch'] in branches: - branch_name = repository['default_branch'] - - contents = gl_client.getrawfile(repository['id'], branch_name, path) - if contents is False: - return None - - return contents - - def list_field_values(self, field_name, limit=None): - if field_name == 'refs': - branches = self.list_field_values('branch_name') - tags = self.list_field_values('tag_name') - - return ([{'kind': 'branch', 'name': b} for b in branches] + - [{'kind': 'tag', 'name': t} for t in tags]) - - gl_client = self._get_authorized_client() - repo = gl_client.getproject(self.config['build_source']) - if repo is False: - return [] - - if field_name == 'tag_name': - tags = gl_client.getrepositorytags(repo['id']) - if tags is False: - return [] - - if limit: - tags = tags[0:limit] - - return [tag['name'] for tag in tags] - - if field_name == 'branch_name': - branches = gl_client.getbranches(repo['id']) - if branches is False: - return [] - - if limit: - branches = branches[0:limit] - - return [branch['name'] for branch in branches] - - return None - - def _prepare_build(self, commit_sha, ref, is_manual): - config = self.config - gl_client = self._get_authorized_client() - - repo = gl_client.getproject(self.config['build_source']) - if repo is False: - raise TriggerStartException('Could not find repository') - - commit = gl_client.getrepositorycommit(repo['id'], commit_sha) - if repo is False: - raise TriggerStartException('Could not find repository') - - committer = None - if 'committer_email' in commit: - try: - [committer] = gl_client.getusers(search=commit['committer_email']) - except ValueError: - committer = None - - try: - [author] = gl_client.getusers(search=commit['author_email']) - except ValueError: - author = None - - metadata = { - 'commit': commit['id'], - 'ref': ref, - 'default_branch': repo['default_branch'], - 'git_url': repo['ssh_url_to_repo'], - 'commit_info': { - 'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'], - 'message': commit['message'], - 'date': commit['committed_date'], - }, - } - - if committer is not None: - metadata['commit_info']['committer'] = { - 'username': committer['username'], - 'avatar_url': committer['avatar_url'], - 'url': gl_client.host + '/' + committer['username'], - } - - if author is not None: - metadata['commit_info']['author'] = { - 'username': author['username'], - 'avatar_url': author['avatar_url'], - 'url': gl_client.host + '/' + author['username'] - } - - prepared = PreparedBuild(self.trigger) - prepared.tags_from_ref(ref, repo['default_branch']) - prepared.name_from_sha(commit['id']) - prepared.subdirectory = config['subdir'] - prepared.metadata = metadata - prepared.is_manual = is_manual - - return prepared - - def handle_trigger_request(self, request): - payload = request.get_json() - if not payload: - raise SkipRequestException() - - logger.debug('GitLab trigger payload %s', payload) - - if not payload.get('commits'): - raise SkipRequestException() - - commit = payload['commits'][0] - commit_message = commit['message'] - if should_skip_commit(commit_message): - raise SkipRequestException() - - ref = payload['ref'] - raise_if_skipped(self.config, ref) - - return self._prepare_build(commit['id'], ref, False) - - def manual_start(self, run_parameters=None): - gl_client = self._get_authorized_client() - - repo = gl_client.getproject(self.config['build_source']) - if repo is False: - raise TriggerStartException('Could not find repository') - - def get_tag_sha(tag_name): - tags = gl_client.getrepositorytags(repo['id']) - if tags is False: - raise TriggerStartException('Could not find tags') - - for tag in tags: - if tag['name'] == tag_name: - return tag['commit']['id'] - - raise TriggerStartException('Could not find commit') - - def get_branch_sha(branch_name): - branch = gl_client.getbranch(repo['id'], branch_name) - if branch is False: - raise TriggerStartException('Could not find branch') - - return branch['commit']['id'] - - # Find the branch or tag to build. - (commit_sha, ref) = _determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, - repo['default_branch']) - - - return self._prepare_build(commit_sha, ref, True) - - def get_repository_url(self): - gl_client = self._get_authorized_client() - repository = gl_client.getproject(self.config['build_source']) - if repository is False: - return None - - return '%s/%s' % (gl_client.host, repository['path_with_namespace']) - diff --git a/endpoints/v1/index.py b/endpoints/v1/index.py index 4d701b918..b17a42f50 100644 --- a/endpoints/v1/index.py +++ b/endpoints/v1/index.py @@ -9,7 +9,7 @@ from data import model from app import app, authentication, userevents, storage from auth.auth import process_auth, generate_signed_token from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token -from util.names import parse_repository_name +from util.names import parse_repository_name, REPOSITORY_NAME_REGEX from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission, ReadRepositoryPermission, CreateRepositoryPermission, repository_read_grant, repository_write_grant) @@ -173,6 +173,10 @@ def update_user(username): @generate_headers(scope=GrantType.WRITE_REPOSITORY, add_grant_for_status=201) @anon_allowed def create_repository(namespace, repository): + # Verify that the repository name is valid. + if not REPOSITORY_NAME_REGEX.match(repository): + abort(400, message='Invalid repository name. Repository names cannot contain slashes.') + logger.debug('Looking up repository %s/%s', namespace, repository) repo = model.repository.get_repository(namespace, repository) @@ -232,9 +236,6 @@ def update_images(namespace, repository): # Make sure the repo actually exists. abort(404, message='Unknown repository', issue='unknown-repo') - logger.debug('GCing repository') - model.repository.garbage_collect_repository(namespace, repository) - # Generate a job for each notification that has been added to this repo logger.debug('Adding notifications for repository') @@ -292,16 +293,31 @@ def put_repository_auth(namespace, repository): abort(501, 'Not Implemented', issue='not-implemented') +def conduct_repo_search(username, query, results): + """ Finds matching repositories. """ + def can_read(repo): + if repo.is_public: + return True + + return ReadRepositoryPermission(repo.namespace_user.username, repo.name).can() + + only_public = username is None + matching_repos = model.repository.get_sorted_matching_repositories(query, only_public, can_read, + limit=5) + + for repo in matching_repos: + results.append({ + 'name': repo.name, + 'description': repo.description, + 'is_public': repo.is_public, + 'href': '/repository/' + repo.namespace_user.username + '/' + repo.name + }) + + @v1_bp.route('/search', methods=['GET']) @process_auth @anon_protect def get_search(): - def result_view(repo): - return { - "name": repo.namespace_user.username + '/' + repo.name, - "description": repo.description - } - query = request.args.get('q') username = None @@ -309,14 +325,9 @@ def get_search(): if user is not None: username = user.username + results = [] if query: - matching = model.repository.get_matching_repositories(query, username) - else: - matching = [] - - results = [result_view(repo) for repo in matching - if (repo.visibility.name == 'public' or - ReadRepositoryPermission(repo.namespace_user.username, repo.name).can())] + conduct_repo_search(username, query, results) data = { "query": query, diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py index f38e2f66d..428c6f19c 100644 --- a/endpoints/v1/registry.py +++ b/endpoints/v1/registry.py @@ -193,11 +193,11 @@ def put_image_layer(namespace, repository, image_id): repo_image = model.image.get_repo_image_extended(namespace, repository, image_id) try: logger.debug('Retrieving image data') - json_data = model.image.get_image_json(repo_image) - except (IOError, AttributeError): + uuid = repo_image.storage.uuid + json_data = repo_image.v1_json_metadata + except (AttributeError): logger.exception('Exception when retrieving image data') - abort(404, 'Image %(image_id)s not found', issue='unknown-image', - image_id=image_id) + abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) uuid = repo_image.storage.uuid layer_path = store.v1_image_layer_path(uuid) @@ -241,15 +241,15 @@ def put_image_layer(namespace, repository, image_id): logger.exception('Exception when writing image data') abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id) + # Save the size of the image. + model.image.set_image_size(image_id, namespace, repository, size_info.compressed_size, + size_info.uncompressed_size) + # Append the computed checksum. csums = [] csums.append('sha256:{0}'.format(h.hexdigest())) try: - # Save the size of the image. - model.image.set_image_size(image_id, namespace, repository, size_info.compressed_size, - size_info.uncompressed_size) - if requires_tarsum: tmp.seek(0) csums.append(checksums.compute_tarsum(tmp, json_data)) @@ -315,7 +315,7 @@ def put_image_checksum(namespace, repository, image_id): abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) logger.debug('Looking up repo layer data') - if not model.image.has_image_json(repo_image): + if not repo_image.v1_json_metadata: abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) logger.debug('Marking image path') @@ -355,21 +355,17 @@ def get_image_json(namespace, repository, image_id, headers): logger.debug('Looking up repo image') repo_image = model.image.get_repo_image_extended(namespace, repository, image_id) - - logger.debug('Looking up repo layer data') - try: - data = model.image.get_image_json(repo_image) - except (IOError, AttributeError): + if repo_image is None: flask_abort(404) logger.debug('Looking up repo layer size') size = repo_image.storage.image_size - - headers['Content-Type'] = 'application/json' if size is not None: + # Note: X-Docker-Size is optional and we *can* end up with a NULL image_size, + # so handle this case rather than failing. headers['X-Docker-Size'] = str(size) - response = make_response(data, 200) + response = make_response(repo_image.v1_json_metadata, 200) response.headers.extend(headers) return response @@ -472,7 +468,8 @@ def put_image_json(namespace, repository, image_id): abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s', issue='invalid-request', image_id=image_id, parent_id=parent_id) - if not image_is_uploading(repo_image) and model.image.has_image_json(repo_image): + logger.debug('Checking if image already exists') + if repo_image.v1_json_metadata and not image_is_uploading(repo_image): exact_abort(409, 'Image already exists') set_uploading_flag(repo_image, True) diff --git a/endpoints/v1/tag.py b/endpoints/v1/tag.py index 346d559d2..865644e2e 100644 --- a/endpoints/v1/tag.py +++ b/endpoints/v1/tag.py @@ -5,7 +5,7 @@ import json from flask import abort, request, jsonify, make_response, session from app import app -from util.names import parse_repository_name +from util.names import TAG_ERROR, TAG_REGEX, parse_repository_name from auth.auth import process_auth from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission) @@ -60,6 +60,9 @@ def put_tag(namespace, repository, tag): permission = ModifyRepositoryPermission(namespace, repository) if permission.can(): + if not TAG_REGEX.match(tag): + abort(400, TAG_ERROR) + docker_image_id = json.loads(request.data) model.tag.create_or_update_tag(namespace, repository, tag, docker_image_id) @@ -83,8 +86,6 @@ def delete_tag(namespace, repository, tag): if permission.can(): model.tag.delete_tag(namespace, repository, tag) - model.repository.garbage_collect_repository(namespace, repository) - return make_response('Deleted', 200) abort(403) diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index 917de3621..3e8754dd5 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -383,10 +383,10 @@ def _generate_and_store_manifest(namespace, repo_name, tag_name): builder = SignedManifestBuilder(namespace, repo_name, tag_name) # Add the leaf layer - builder.add_layer(image.storage.checksum, __get_and_backfill_image_metadata(image)) + builder.add_layer(image.storage.checksum, image.v1_json_metadata) for parent in parents: - builder.add_layer(parent.storage.checksum, __get_and_backfill_image_metadata(parent)) + builder.add_layer(parent.storage.checksum, parent.v1_json_metadata) # Sign the manifest with our signing key. manifest = builder.build(docker_v2_signing_key) @@ -394,15 +394,3 @@ def _generate_and_store_manifest(namespace, repo_name, tag_name): manifest.digest, manifest.bytes) return manifest_row - - -def __get_and_backfill_image_metadata(image): - image_metadata = image.v1_json_metadata - if image_metadata is None: - logger.warning('Loading metadata from storage for image id: %s', image.id) - - image.v1_json_metadata = model.image.get_image_json(image) - logger.info('Saving backfilled metadata for image id: %s', image.id) - image.save() - - return image_metadata diff --git a/endpoints/verbs.py b/endpoints/verbs.py index 560acbdd6..5ba858c98 100644 --- a/endpoints/verbs.py +++ b/endpoints/verbs.py @@ -27,7 +27,7 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag store = Storage(app) def get_image_json(image): - return json.loads(model.image.get_image_json(image)) + return json.loads(image.v1_json_metadata) def get_next_image(): for current_image in image_list: @@ -113,7 +113,7 @@ def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None): abort(404) # Lookup the tag's image and storage. - repo_image = model.image.get_repo_image_extended(namespace, repository, tag_image.docker_image_id) + repo_image = model.image.get_repo_image(namespace, repository, tag_image.docker_image_id) if not repo_image: abort(404) @@ -121,7 +121,8 @@ def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None): image_json = None if checker is not None: - image_json = json.loads(model.image.get_image_json(repo_image)) + image_json = json.loads(repo_image.v1_json_metadata) + if not checker(image_json): logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb) abort(404) @@ -187,7 +188,7 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker= # Load the image's JSON layer. if not image_json: - image_json = json.loads(model.image.get_image_json(repo_image)) + image_json = json.loads(repo_image.v1_json_metadata) # Calculate a synthetic image ID. synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest() diff --git a/endpoints/web.py b/endpoints/web.py index 154483faf..f7450b599 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -21,8 +21,12 @@ from util.cache import no_cache from endpoints.common import common_login, render_page_template, route_show_if, param_required from endpoints.decorators import anon_protect from endpoints.csrf import csrf_protect, generate_csrf_token, verify_csrf -from endpoints.trigger import (CustomBuildTrigger, BitbucketBuildTrigger, TriggerProviderException, - BuildTriggerHandler) + +from buildtrigger.customhandler import CustomBuildTrigger +from buildtrigger.bitbuckethandler import BitbucketBuildTrigger +from buildtrigger.triggerutil import TriggerProviderException +from buildtrigger.basehandler import BuildTriggerHandler + from util.names import parse_repository_name, parse_repository_name_and_tag from util.useremails import send_email_changed from util.systemlogs import build_logs_archive diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index 836b2fa9b..1b3d23f23 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -9,8 +9,9 @@ from auth.permissions import ModifyRepositoryPermission from util.invoice import renderInvoiceToHtml from util.useremails import send_invoice_email, send_subscription_change, send_payment_failed from util.http import abort -from endpoints.trigger import (BuildTriggerHandler, ValidationRequestException, - SkipRequestException, InvalidPayloadException) +from buildtrigger.basehandler import BuildTriggerHandler +from buildtrigger.triggerutil import (ValidationRequestException, SkipRequestException, + InvalidPayloadException) from endpoints.building import start_build diff --git a/formats/tarimageformatter.py b/formats/tarimageformatter.py index 38d3fb3ab..ca46bca12 100644 --- a/formats/tarimageformatter.py +++ b/formats/tarimageformatter.py @@ -44,4 +44,4 @@ class TarImageFormatter(object): """ Returns TAR file header data for a folder with the given name. """ info = tarfile.TarInfo(name=name) info.type = tarfile.DIRTYPE - return info.tobuf() \ No newline at end of file + return info.tobuf() diff --git a/initdb.py b/initdb.py index 93b2e655d..d08822eb5 100644 --- a/initdb.py +++ b/initdb.py @@ -90,9 +90,7 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map): # Write some data for the storage. if os.environ.get('WRITE_STORAGE_FILES'): storage_paths = StoragePaths() - paths = [storage_paths.image_json_path, - storage_paths.image_ancestry_path, - storage_paths.image_layer_path] + paths = [storage_paths.v1_image_layer_path] for path_builder in paths: path = path_builder(new_image.storage.uuid) diff --git a/local-docker.sh b/local-docker.sh index 530ff2dae..d6fd55cf1 100755 --- a/local-docker.sh +++ b/local-docker.sh @@ -7,7 +7,7 @@ REPO=quay.io/quay/quay-dev d () { docker build -t $REPO -f dev.df . - docker -- run --rm -it --net=host -v $(pwd)/..:/src $REPO $* + docker -- run --rm -v /var/run/docker.sock:/run/docker.sock -it --net=host -v $(pwd)/..:/src $REPO $* } case $1 in @@ -23,6 +23,13 @@ notifications) test) d bash /src/quay/local-test.sh ;; +initdb) + rm -f test/data/test.db + d /venv/bin/python initdb.py + ;; +fulldbtest) + d bash /src/quay/test/fulldbtest.sh + ;; *) echo "unknown option" exit 1 diff --git a/local-test.sh b/local-test.sh index ba015d773..bab484c9c 100755 --- a/local-test.sh +++ b/local-test.sh @@ -1 +1,7 @@ -TEST=true TROLLIUSDEBUG=1 python -m unittest discover -f +set -e + +export TEST=true +export TROLLIUSDEBUG=1 + +python -m unittest discover -f +python -m test.registry_tests -f diff --git a/registry.py b/registry.py index 1582a2879..df868242c 100644 --- a/registry.py +++ b/registry.py @@ -1,5 +1,6 @@ import logging import logging.config +import os from app import app as application @@ -9,5 +10,8 @@ import endpoints.decorated from endpoints.v1 import v1_bp from endpoints.v2 import v2_bp +if os.environ.get('DEBUGLOG') == 'true': + logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) + application.register_blueprint(v1_bp, url_prefix='/v1') application.register_blueprint(v2_bp, url_prefix='/v2') diff --git a/requirements-nover.txt b/requirements-nover.txt index 4e0b04d33..29628f1ba 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -38,7 +38,7 @@ git+https://github.com/DevTable/pygithub.git git+https://github.com/DevTable/container-cloud-config.git git+https://github.com/coreos/mockldap.git git+https://github.com/coreos/py-bitbucket.git -git+https://github.com/coreos/pyapi-gitlab.git +git+https://github.com/coreos/pyapi-gitlab.git@timeout git+https://github.com/coreos/resumablehashlib.git git+https://github.com/DevTable/python-etcd.git@sslfix gipc @@ -55,4 +55,4 @@ pyjwt toposort pyjwkest rfc3987 -pyjwkest +jsonpath-rw \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 598558965..83025b635 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ blinker==1.3 boto==2.38.0 cachetools==1.0.3 cffi==1.1.2 -cryptography==0.9.2 +cryptography==1.0.2 debtcollector==0.5.0 enum34==1.0.4 Flask==0.10.1 @@ -32,6 +32,7 @@ iso8601==0.1.10 itsdangerous==0.24 Jinja2==2.7.3 jsonschema==2.5.1 +jsonpath-rw==1.4.0 Mako==1.0.1 marisa-trie==0.7.2 MarkupSafe==0.23 @@ -97,7 +98,8 @@ git+https://github.com/DevTable/pygithub.git git+https://github.com/DevTable/container-cloud-config.git git+https://github.com/coreos/mockldap.git git+https://github.com/coreos/py-bitbucket.git -git+https://github.com/coreos/pyapi-gitlab.git +git+https://github.com/coreos/pyapi-gitlab.git@timeout git+https://github.com/coreos/resumablehashlib.git +git+https://github.com/coreos/mockldap.git git+https://github.com/DevTable/python-etcd.git@sslfix git+https://github.com/NateFerrero/oauth2lib.git diff --git a/static/css/pages/team-view.css b/static/css/pages/team-view.css index 0e9a8d7a0..cf80b8004 100644 --- a/static/css/pages/team-view.css +++ b/static/css/pages/team-view.css @@ -2,6 +2,12 @@ padding: 20px; } +.team-view .team-title { + vertical-align: middle; + margin-right: 10px; + color: #ccc; +} + .team-view .team-name { vertical-align: middle; margin-left: 6px; diff --git a/static/directives/build-logs-view.html b/static/directives/build-logs-view.html index a26f3eafc..b1cf1aebc 100644 --- a/static/directives/build-logs-view.html +++ b/static/directives/build-logs-view.html @@ -24,34 +24,41 @@ please check for JavaScript or networking issues and contact support. - - (Waiting for build to start) - +
+ Refreshing Build Status... + +
-
-
- -
- -
-
- -
-
- -
-
+
+ + (Waiting for build to start) + - -
-
- - - - +
+
+ +
+ +
+
+ +
+
+ +
+
+ + +
+
+ + + + +
diff --git a/static/directives/tag-operations-dialog.html b/static/directives/tag-operations-dialog.html index a769d5fa1..901a7792c 100644 --- a/static/directives/tag-operations-dialog.html +++ b/static/directives/tag-operations-dialog.html @@ -34,7 +34,7 @@
{{ revertTagInfo.tag.name }} to image {{ revertTagInfo.image_id.substr(0, 12) }}?
-
\ No newline at end of file +
diff --git a/static/directives/teams-manager.html b/static/directives/teams-manager.html index 141892ca3..72b459257 100644 --- a/static/directives/teams-manager.html +++ b/static/directives/teams-manager.html @@ -23,15 +23,20 @@
- +
+ +