From 011490d36d261fea44e5bb4a01f42f1c79d96a9b Mon Sep 17 00:00:00 2001 From: jakedt Date: Tue, 25 Feb 2014 19:39:43 -0500 Subject: [PATCH] Refactor a lot of the build create code out into a common method. Add an endpoint for manually starting triggers. --- endpoints/api.py | 71 +++++++++++++++++++++++++-------------- endpoints/common.py | 48 ++++++++++++++++++++++++++- endpoints/trigger.py | 77 +++++++++++++++++++++++++++++-------------- endpoints/webhooks.py | 38 +++------------------ 4 files changed, 150 insertions(+), 84 deletions(-) diff --git a/endpoints/api.py b/endpoints/api.py index dfca74d58..4c2f1ce6f 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -12,7 +12,6 @@ from collections import defaultdict from urllib import quote from data import model -from data.queue import dockerfile_build_queue from data.plans import PLANS, get_plan from app import app from util.email import (send_confirmation_email, send_recovery_email, @@ -28,9 +27,11 @@ from auth.permissions import (ReadRepositoryPermission, OrganizationMemberPermission, ViewTeamPermission, UserPermission) -from endpoints.common import common_login, get_route_data, truthy_param +from endpoints.common import (common_login, get_route_data, truthy_param, + start_build) from endpoints.trigger import (BuildTrigger, TriggerActivationException, - TriggerDeactivationException, EmptyRepositoryException) + TriggerDeactivationException, + EmptyRepositoryException) from util.cache import cache_control from datetime import datetime, timedelta @@ -1247,28 +1248,10 @@ def request_repo_build(namespace, repository): # Start the build. repo = model.get_repository(namespace, repository) - token = model.create_access_token(repo, 'write') display_name = user_files.get_file_checksum(dockerfile_id) - logger.debug('**********Md5: %s' % display_name) - host = urlparse.urlparse(request.url).netloc - repo_tag_base = '%s/%s/%s' % (host, repo.namespace, repo.name) - job_config = { - 'docker_tags': ['latest'], - 'build_subdir': '', - 'repository': repo_tag_base, - } - build_request = model.create_repository_build(repo, token, job_config, - dockerfile_id, display_name) - dockerfile_build_queue.put(json.dumps({ - 'build_uuid': build_request.uuid, - 'namespace': namespace, - 'repository': repository, - }), retries_remaining=1) - - log_action('build_dockerfile', namespace, - {'repo': repository, 'namespace': namespace, - 'fileid': dockerfile_id}, repo=repo) + build_request = start_build(repo, dockerfile_id, ['latest'], display_name, + '', True) resp = jsonify(build_status_view(build_request, True)) repo_string = '%s/%s' % (namespace, repository) @@ -1441,8 +1424,8 @@ def activate_build_trigger(namespace, repository, trigger_uuid): try: repository_path = '%s/%s' % (trigger.repository.namespace, trigger.repository.name) - path = url_for('webhooks.build_trigger_webhook', repository=repository_path, - trigger_uuid=trigger.uuid) + path = url_for('webhooks.build_trigger_webhook', + repository=repository_path, trigger_uuid=trigger.uuid) authed_url = _prepare_webhook_url(app.config['URL_SCHEME'], '$token', token.code, app.config['URL_HOST'], path) @@ -1471,6 +1454,44 @@ def activate_build_trigger(namespace, repository, trigger_uuid): abort(403) # Permission denied +@api.route('/repository//trigger//start', + methods=['POST']) +@api_login_required +@parse_repository_name +def manually_start_build_trigger(namespace, repository, trigger_uuid): + permission = AdministerRepositoryPermission(namespace, repository) + if permission.can(): + try: + trigger = model.get_build_trigger(namespace, repository, trigger_uuid) + except model.InvalidBuildTriggerException: + abort(404) + return + + handler = BuildTrigger.get_trigger_for_service(trigger.service.name) + existing_config_dict = json.loads(trigger.config) + if handler.is_active(existing_config_dict): + abort(400) + return + + specs = handler.manual_start(trigger.auth_token, + json.loads(trigger.config)) + dockerfile_id, tags, name, subdir = specs + + repo = model.get_repository(namespace, repository) + + build_request = start_build(repo, dockerfile_id, tags, name, subdir, True) + + resp = jsonify(build_status_view(build_request, True)) + repo_string = '%s/%s' % (namespace, repository) + resp.headers['Location'] = url_for('api.get_repo_build_status', + repository=repo_string, + build_uuid=build_request.uuid) + resp.status_code = 201 + return resp + + abort(403) # Permission denied + + @api.route('/repository//trigger//builds', methods=['GET']) @api_login_required diff --git a/endpoints/common.py b/endpoints/common.py index 61051237e..c479a3566 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -1,12 +1,15 @@ import logging import os import base64 +import urlparse +import json -from flask import session, make_response, render_template +from flask import session, make_response, render_template, request from flask.ext.login import login_user, UserMixin from flask.ext.principal import identity_changed from data import model +from data.queue import dockerfile_build_queue from app import app, login_manager from auth.permissions import QuayDeferredPermissionUser @@ -115,3 +118,46 @@ def render_page_template(name, **kwargs): **kwargs)) resp.headers['X-FRAME-OPTIONS'] = 'DENY' return resp + + +def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, + trigger=None): + host = urlparse.urlparse(request.url).netloc + repo_path = '%s/%s/%s' % (host, repository.namespace, repository.name) + + token = model.create_access_token(repository, 'write') + logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', + build_name, repo_path, tags, dockerfile_id) + + job_config = { + 'docker_tags': tags, + 'repository': repo_path, + 'build_subdir': subdir, + } + build_request = model.create_repository_build(repository, token, job_config, + dockerfile_id, build_name, + trigger) + + dockerfile_build_queue.put(json.dumps({ + 'build_uuid': build_request.uuid, + 'namespace': repository.namespace, + 'repository': repository.name, + }), retries_remaining=1) + + metadata = { + 'repo': repository.name, + 'namespace': repository.namespace, + 'fileid': dockerfile_id, + 'manual': manual, + } + + if trigger: + metadata['trigger_id'] = trigger.uuid + metadata['config'] = json.loads(trigger.config) + metadata['service'] = trigger.service.name + + model.log_action('build_dockerfile', repository.namespace, + ip=request.remote_addr, metadata=metadata, + repository=repository) + + return build_request \ No newline at end of file diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 9dc266394..41c32045a 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -86,6 +86,12 @@ class BuildTrigger(object): """ raise NotImplementedError + def manual_start(self, auth_token, config): + """ + Manually creates a repository build for this trigger. + """ + raise NotImplementedError + @classmethod def service_name(cls): """ @@ -137,6 +143,7 @@ class GithubBuildTrigger(BuildTrigger): try: hook = to_add_webhook.create_hook('web', webhook_config) config['hook_id'] = hook.id + config['master_branch'] = to_add_webhook.master_branch except GithubException: msg = 'Unable to create webhook on repository: %s' raise TriggerActivationException(msg % new_build_source) @@ -206,27 +213,10 @@ class GithubBuildTrigger(BuildTrigger): msg = 'Unable to list contents of repository: %s' % source raise EmptyRepositoryException(msg) - def handle_trigger_request(self, request, auth_token, config): - payload = request.get_json() - - if 'zen' in payload: - raise ValidationRequestException() - - logger.debug('Payload %s', payload) - ref = payload['ref'] - commit_sha = payload['head_commit']['id'] - short_sha = commit_sha[0:7] - - gh_client = self._get_client(auth_token) - - repo_full_name = '%s/%s' % (payload['repository']['owner']['name'], - payload['repository']['name']) - repo = gh_client.get_repo(repo_full_name) - - logger.debug('Github repo: %s', repo) - + @staticmethod + def _prepare_build(config, repo, commit_sha, build_name, ref): # Prepare the download and upload URLs - archive_link = repo.get_archive_link('zipball', short_sha) + archive_link = repo.get_archive_link('zipball', commit_sha) download_archive = client.get(archive_link, stream=True) zipball_subdir = '' @@ -243,9 +233,9 @@ class GithubBuildTrigger(BuildTrigger): logger.debug('Successfully prepared job') # compute the tag(s) - pushed_branch = ref.split('/')[-1] - tags = {pushed_branch} - if pushed_branch == repo.master_branch: + branch = ref.split('/')[-1] + tags = {branch} + if branch == repo.master_branch: tags.add('latest') logger.debug('Pushing to tags: %s' % tags) @@ -254,4 +244,43 @@ class GithubBuildTrigger(BuildTrigger): joined_subdir = os.path.join(zipball_subdir, repo_subdir) logger.debug('Final subdir: %s' % joined_subdir) - return dockerfile_id, list(tags), short_sha, joined_subdir + return dockerfile_id, list(tags), build_name, joined_subdir + + @staticmethod + def get_display_name(sha): + return sha[0:7] + + def handle_trigger_request(self, request, auth_token, config): + payload = request.get_json() + + if 'zen' in payload: + raise ValidationRequestException() + + logger.debug('Payload %s', payload) + ref = payload['ref'] + commit_sha = payload['head_commit']['id'] + short_sha = GithubBuildTrigger.get_display_name(commit_sha) + + gh_client = self._get_client(auth_token) + + repo_full_name = '%s/%s' % (payload['repository']['owner']['name'], + payload['repository']['name']) + repo = gh_client.get_repo(repo_full_name) + + logger.debug('Github repo: %s', repo) + + return GithubBuildTrigger._prepare_build(config, repo, commit_sha, + short_sha, ref) + + def manual_start(self, auth_token, config): + source = config['build_source'] + subdir = config['subdir'] + + gh_client = self._get_client(auth_token) + repo = gh_client.get_repo(source) + master = repo.get_branch(repo.master_branch) + master_sha = master.commit.sha + short_sha = GithubBuildTrigger.get_display_name(master_sha) + ref = 'refs/heads/%s' % repo.master_branch + + return self._prepare_build(config, repo, master_sha, short_sha, ref) \ No newline at end of file diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index 830649a59..d92e7095e 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -1,6 +1,5 @@ import logging import stripe -import urlparse import json from flask import request, make_response, Blueprint @@ -14,6 +13,7 @@ from util.email import send_invoice_email from util.names import parse_repository_name from util.http import abort from endpoints.trigger import BuildTrigger, ValidationRequestException +from endpoints.common import start_build logger = logging.getLogger(__name__) @@ -65,46 +65,16 @@ def build_trigger_webhook(namespace, repository, trigger_uuid): logger.debug('Passing webhook request to handler %s', handler) config_dict = json.loads(trigger.config) try: - specs = handler.handle_trigger_request(request, trigger.auth_token, config_dict) + specs = handler.handle_trigger_request(request, trigger.auth_token, + config_dict) dockerfile_id, tags, name, subdir = specs except ValidationRequestException: # This was just a validation request, we don't need to build anything return make_response('Okay') - host = urlparse.urlparse(request.url).netloc - repo_path = '%s/%s/%s' % (host, trigger.repository.namespace, - trigger.repository.name) - - token = model.create_access_token(trigger.repository, 'write') - logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s', - name, repo_path, tags, dockerfile_id) - - job_config = { - 'docker_tags': tags, - 'repository': repo, - 'build_subdir': subdir, - } - build_request = model.create_repository_build(trigger.repository, token, - job_config, dockerfile_id, - name, trigger) - - dockerfile_build_queue.put(json.dumps({ - 'build_uuid': build_request.uuid, - 'namespace': namespace, - 'repository': repository, - }), retries_remaining=1) - - metadata = { - 'repo': repository, - 'namespace': namespace, - 'trigger_id': trigger_uuid, - 'fileid': dockerfile_id, - 'config': config_dict, - 'service': trigger.service.name - } repo = model.get_repository(namespace, repository) - model.log_action('build_dockerfile', namespace, ip=request.remote_addr, metadata=metadata, repository=repo) + start_build(repo, dockerfile_id, tags, name, subdir, False, trigger) return make_response('Okay')