From af31bde997a2751a29c20c7f1b664cff9874da66 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 18 Jul 2014 15:58:18 -0400 Subject: [PATCH] Add support for the remaining events to the frontend and the backend --- endpoints/common.py | 47 ++++++++- endpoints/index.py | 20 +--- endpoints/notificationevent.py | 165 ++++++++++++++++++++++++++++---- endpoints/notificationmethod.py | 4 +- initdb.py | 2 + static/js/app.js | 28 +++++- workers/dockerfilebuild.py | 43 ++++++++- 7 files changed, 269 insertions(+), 40 deletions(-) diff --git a/endpoints/common.py b/endpoints/common.py index e404bde05..e3e8533db 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -9,7 +9,7 @@ from flask.ext.principal import identity_changed from random import SystemRandom from data import model -from app import app, login_manager, dockerfile_build_queue +from app import app, login_manager, dockerfile_build_queue, notification_queue from auth.permissions import QuayDeferredPermissionUser from auth import scopes from endpoints.api.discovery import swagger_route_data @@ -21,6 +21,7 @@ from external_libraries import get_external_javascript, get_external_css import features logger = logging.getLogger(__name__) +profile = logging.getLogger('application.profiler') route_data = None @@ -207,6 +208,7 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, 'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None }), retries_remaining=1) + # Add the build to the repo's log. metadata = { 'repo': repository.name, 'namespace': repository.namespace, @@ -223,4 +225,47 @@ def start_build(repository, dockerfile_id, tags, build_name, subdir, manual, ip=request.remote_addr, metadata=metadata, repository=repository) + # Add notifications for the build queue. + profile.debug('Adding notifications for repository') + event_data = { + 'build_id': build_request.uuid, + 'build_name': build_name, + 'docker_tags': tags, + 'is_manual': manual, + 'trigger_id': trigger.uuid, + 'trigger_kind': trigger.service.name + } + + spawn_notification(repository, 'build_queued', event_data, + subpage='build?current=' % build_request.uuid, + pathargs=['build', build_request.uuid]) return build_request + + +def spawn_notification(repository, event_name, extra_data={}, subpage=None, pathargs=[]): + homepage = 'https://quay.io/repository/%s' % repo_string + if subpage: + homepage = homepage + subpage + + repo_string = '%s/%s' % (repo.namespace, repo.name) + event_data = { + 'repository': repo_string, + 'namespace': repo.namespace, + 'name': repo.name, + 'docker_url': 'quay.io/%s' % repo_string, + 'homepage': homepage, + 'visibility': repo.visibility.name + } + + event_data.update(extra_data) + + notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name) + for notification in notifications: + notification_data = { + 'notification_id': notification.id, + 'repository_id': repository.id, + 'event_data': event_data + } + + path = [namespace, repository, 'notification', event_name] + pathargs + notification_queue.put(path, json.dumps(notification_data)) diff --git a/endpoints/index.py b/endpoints/index.py index a410f9b38..c6cc68020 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -17,6 +17,7 @@ from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission, ReadRepositoryPermission, CreateRepositoryPermission) from util.http import abort +from endpoints.common import spawn_notification logger = logging.getLogger(__name__) @@ -307,7 +308,7 @@ def update_images(namespace, repository): 'action': 'pushed_repo', 'repository': repository, 'namespace': namespace - } + } event = userevents.get_event(username) event.publish_event_data('docker-cli', user_data) @@ -318,28 +319,13 @@ def update_images(namespace, repository): # Generate a job for each notification that has been added to this repo profile.debug('Adding notifications for repository') - repo_string = '%s/%s' % (namespace, repository) event_data = { - 'repository': repo_string, - 'namespace': namespace, - 'name': repository, - 'docker_url': 'quay.io/%s' % repo_string, - 'homepage': 'https://quay.io/repository/%s' % repo_string, - 'visibility': repo.visibility.name, 'updated_tags': updated_tags, 'pushed_image_count': len(image_with_checksums), 'pruned_image_count': num_removed } - notifications = model.list_repo_notifications(namespace, repository, event_name='repo_push') - for notification in notifications: - notification_data = { - 'notification_id': notification.id, - 'repository_id': repository.id, - 'event_data': event_data - } - notification_queue.put([namespace, repository, 'repo_push'], json.dumps(notification_data)) - + spawn_notification(repo, 'repo_push', event_data) return make_response('Updated', 204) abort(403) diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index 215962fbd..e453da669 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -13,13 +13,13 @@ class NotificationEvent(object): def __init__(self): pass - def get_summary(self, notification_data): + def get_summary(self, event_data, notification_data): """ Returns a human readable one-line summary for the given notification data. """ raise NotImplementedError - def get_message(self, notification_data): + def get_message(self, event_data, notification_data): """ Returns a human readable HTML message for the given notification data. """ @@ -53,19 +53,27 @@ class RepoPushEvent(NotificationEvent): def event_name(cls): return 'repo_push' - def get_summary(self, notification_data): - return 'Repository %s updated' % (notification_data['event_data']['repository']) + def get_summary(self, event_data, notification_data): + return 'Repository %s updated' % (event_data['repository']) - def get_message(self, notification_data): - event_data = notification_data['event_data'] + def get_message(self, event_data, notification_data): if not event_data.get('updated_tags', []): - return '%s images pushed for repository %s (%s)' % (event_data['pushed_image_count'], - event_data['repository'], event_data['homepage']) + html = """ + Repository %s has been updated via a push. + """ % (event_data['homepage'], + event_data['repository']) + else: + html = """ + Repository %s has been updated via a push. +

+ Tags Updated: %s + """ % (event_data['homepage'], + event_data['repository'], + event_data['updated_tags']) - return 'Tags %s updated for repository %s (%s)' % (event_data['updated_tags'], - event_data['repository'], event_data['homepage']) + return html - def get_sample_data(self, repository=None): + def get_sample_data(self, repository): repo_string = '%s/%s' % (repository.namespace, repository.name) event_data = { 'repository': repo_string, @@ -81,22 +89,117 @@ class RepoPushEvent(NotificationEvent): return event_data +class BuildQueueEvent(NotificationEvent): + @classmethod + def event_name(cls): + return 'build_queued' + + def get_sample_data(self, repository): + build_uuid = 'fake-build-id' + repo_string = '%s/%s' % (repository.namespace, repository.name) + event_data = { + 'repository': repo_string, + 'namespace': repository.namespace, + 'name': repository.name, + 'docker_url': 'quay.io/%s' % repo_string, + 'homepage': 'https://quay.io/repository/%s/build/%s' % (repo_string, build_uuid), + 'is_manual': False, + 'build_id': build_uuid, + 'build_name': 'some-fake-build', + 'docker_tags': ['latest', 'foo', 'bar'], + 'trigger_kind': 'GitHub' + } + return event_data + + def get_summary(self, event_data, notification_data): + return 'Build queued for repository %s' % (event_data['repository']) + + def get_message(self, event_data, notification_data): + is_manual = event_data['is_manual'] + if is_manual: + html = """ + A new build has been manually queued to start on repository %s. +

+ Build ID: %s + """ % (event_data['homepage'], event_data['repository'], event_data['build_id']) + else: + html = """ + A new build has been queued via a %s trigger to start on repository %s. +

+ Build ID: %s + """ % (event_data['homepage'], event_data['repository'], + event_data['trigger_kind'], event_data['build_id']) + + return html + + + class BuildStartEvent(NotificationEvent): @classmethod def event_name(cls): return 'build_start' - def get_sample_data(self, repository=None): - pass + def get_sample_data(self, repository): + build_uuid = 'fake-build-id' + repo_string = '%s/%s' % (repository.namespace, repository.name) + event_data = { + 'repository': repo_string, + 'namespace': repository.namespace, + 'name': repository.name, + 'docker_url': 'quay.io/%s' % repo_string, + 'homepage': 'https://quay.io/repository/%s/build?current=%s' % (repo_string, build_uuid), + 'build_id': build_uuid, + 'build_name': 'some-fake-build', + 'docker_tags': ['latest', 'foo', 'bar'], + 'trigger_kind': 'GitHub' + } + return event_data + + def get_summary(self, event_data, notification_data): + return 'Build started for repository %s' % (event_data['repository']) + def get_message(self, event_data, notification_data): + html = """ + A new build has started on repository %s. +

+ Build ID: %s + """ % (event_data['homepage'], event_data['repository'], event_data['build_id']) + + return html + class BuildSuccessEvent(NotificationEvent): @classmethod def event_name(cls): return 'build_success' - def get_sample_data(self, repository=None): - pass + def get_sample_data(self, repository): + build_uuid = 'fake-build-id' + repo_string = '%s/%s' % (repository.namespace, repository.name) + event_data = { + 'repository': repo_string, + 'namespace': repository.namespace, + 'name': repository.name, + 'docker_url': 'quay.io/%s' % repo_string, + 'homepage': 'https://quay.io/repository/%s/build?current=%s' % (repo_string, build_uuid), + 'build_id': build_uuid, + 'build_name': 'some-fake-build', + 'docker_tags': ['latest', 'foo', 'bar'], + 'trigger_kind': 'GitHub' + } + return event_data + + def get_summary(self, event_data, notification_data): + return 'Build succeeded for repository %s' % (event_data['repository']) + + def get_message(self, event_data, notification_data): + html = """ + A build has finished on repository %s. +

+ Build ID: %s + """ % (event_data['homepage'], event_data['repository'], event_data['build_id']) + + return html class BuildFailureEvent(NotificationEvent): @@ -104,5 +207,33 @@ class BuildFailureEvent(NotificationEvent): def event_name(cls): return 'build_failure' - def get_sample_data(self, repository=None): - pass + def get_sample_data(self, repository): + build_uuid = 'fake-build-id' + repo_string = '%s/%s' % (repository.namespace, repository.name) + event_data = { + 'repository': repo_string, + 'namespace': repository.namespace, + 'name': repository.name, + 'docker_url': 'quay.io/%s' % repo_string, + 'homepage': 'https://quay.io/repository/%s/build?current=%s' % (repo_string, build_uuid), + 'build_id': build_uuid, + 'build_name': 'some-fake-build', + 'docker_tags': ['latest', 'foo', 'bar'], + 'trigger_kind': 'GitHub', + 'error_message': 'This is a fake error message' + } + return event_data + + def get_summary(self, event_data, notification_data): + return 'Build failure for repository %s' % (event_data['repository']) + + def get_message(self, event_data, notification_data): + html = """ + A build has failed on repository %s. +

+ Reason: %s
+ Build ID: %s
+ """ % (event_data['homepage'], event_data['repository'], + event_data['error_message'], event_data['build_id']) + + return html diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index a9ffa78eb..5f0f8247f 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -98,10 +98,10 @@ class EmailMethod(NotificationMethod): if not email: return False - msg = Message(event_handler.get_summary(notification_data), + msg = Message(event_handler.get_summary(notification_data['event_data'], notification_data), sender='support@quay.io', recipients=[email]) - msg.html = event_handler.get_message(notification_data) + msg.html = event_handler.get_message(notification_data['event_data'], notification_data) try: with app.app_context(): diff --git a/initdb.py b/initdb.py index 99a9f0624..c55aceceb 100644 --- a/initdb.py +++ b/initdb.py @@ -242,6 +242,7 @@ def initialize_database(): # NOTE: These MUST be copied over to NotificationKind, since every external # notification can also generate a Quay.io notification. ExternalNotificationEvent.create(name='repo_push') + ExternalNotificationEvent.create(name='build_queued') ExternalNotificationEvent.create(name='build_start') ExternalNotificationEvent.create(name='build_success') ExternalNotificationEvent.create(name='build_failure') @@ -251,6 +252,7 @@ def initialize_database(): ExternalNotificationMethod.create(name='webhook') NotificationKind.create(name='repo_push') + NotificationKind.create(name='build_queued') NotificationKind.create(name='build_start') NotificationKind.create(name='build_success') NotificationKind.create(name='build_failure') diff --git a/static/js/app.js b/static/js/app.js index b1b79ae2f..45e9965cf 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -978,10 +978,15 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'title': 'Push to Repository', 'icon': 'fa-upload' }, + { + 'id': 'build_queued', + 'title': 'Dockerfile Build Queued', + 'icon': 'fa-tasks' + }, { 'id': 'build_start', 'title': 'Dockerfile Build Started', - 'icon': 'fa-tasks' + 'icon': 'fa-circle-o-notch' }, { 'id': 'build_success', @@ -1117,6 +1122,27 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading 'page': function(metadata) { return '/repository/' + metadata.repository; } + }, + 'build_queued': { + 'level': 'info', + 'message': 'A build has been queued for repository {repository}', + 'page': function(metadata) { + return '/repository/' + metadata.repository + '/build?current=' + metadata.build_id; + } + }, + 'build_start': { + 'level': 'info', + 'message': 'A build has been started for repository {repository}', + 'page': function(metadata) { + return '/repository/' + metadata.repository + '/build?current=' + metadata.build_id; + } + }, + 'build_failure': { + 'level': 'error', + 'message': 'A build has failed for repository {repository}', + 'page': function(metadata) { + return '/repository/' + metadata.repository + '/build?current=' + metadata.build_id; + } } }; diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index dbb8d0aa0..a930f6e7b 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -25,6 +25,7 @@ from collections import defaultdict from data import model from workers.worker import Worker, WorkerUnhealthyException, JobException from app import userfiles as user_files, build_logs, sentry, dockerfile_build_queue +from endpoints.common import spawn_notification from util.safetar import safe_extractall from util.dockerfileparse import parse_dockerfile, ParsedDockerfile, serialize_dockerfile @@ -501,6 +502,27 @@ class DockerfileBuildWorker(Worker): build_dir = self._mime_processors[c_type](docker_resource) + # Spawn a notification that the build has started. + event_data = { + 'build_id': repository_build.uuid, + 'build_name': repository_build.display_name, + 'docker_tags': tag_names, + 'trigger_id': repository_build.trigger.uuid, + 'trigger_kind': repository_build.trigger.service.name + } + + spawn_notification(repository, 'build_start', event_data, + subpage='build?current=' % repository_build.uuid, + pathargs=['build', repository_build.uuid]) + + # Setup a handler for spawning failure messages. + def spawn_failure(message, event_data): + event_data['error_message'] = exc.message + spawn_notification(repository, 'build_failure', event_data, + subpage='build?current=' % repository_build.uuid, + pathargs=['build', repository_build.uuid]) + + # Start the build process. try: with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names, access_token, repository_build.uuid, self._cache_size_gb, @@ -541,21 +563,38 @@ class DockerfileBuildWorker(Worker): repository_build.phase = 'complete' repository_build.save() + # Spawn a notification that the build has completed. + spawn_notification(repository, 'build_success', event_data, + subpage='build?current=' % repository_build.uuid, + pathargs=['build', repository_build.uuid]) + except WorkerUnhealthyException as exc: - # Need a separate handler for this so it doesn't get caught by catch all below + # Spawn a notification that the build has failed. + spawn_failure(exc.message, event_data) + + # Raise the exception to the queue. raise exc except JobException as exc: - # Need a separate handler for this so it doesn't get caught by catch all below + # Spawn a notification that the build has failed. + spawn_failure(exc.message, event_data) + + # Raise the exception to the queue. raise exc except Exception as exc: + # Spawn a notification that the build has failed. + spawn_failure(exc.message, event_data) + + # Write the error to the logs. sentry.client.captureException() log_appender('error', build_logs.PHASE) logger.exception('Exception when processing request.') repository_build.phase = 'error' repository_build.save() log_appender(str(exc), build_logs.ERROR) + + # Raise the exception to the queue. raise JobException(str(exc))