From 49801bc2c42362f9fafdbedf2ffc3e4ec7ee7a55 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 31 Jul 2014 13:30:54 -0400 Subject: [PATCH] - Add web hook queue code back in. We'll remove it and turn it off after this CL goes to prod - Make notification lookup always be by repo and its UUID, rather than the internal DB ID - Add the init script for the notification worker --- Dockerfile.web | 3 ++ app.py | 3 ++ conf/init/notificationworker/log/run | 2 ++ conf/init/notificationworker/run | 8 +++++ config.py | 3 ++ data/model/legacy.py | 39 ------------------------- endpoints/api/repositorynotification.py | 7 ++--- endpoints/notificationevent.py | 2 +- endpoints/notificationhelper.py | 15 ++++++---- endpoints/notificationmethod.py | 3 +- workers/notificationworker.py | 8 +++-- 11 files changed, 37 insertions(+), 56 deletions(-) create mode 100755 conf/init/notificationworker/log/run create mode 100755 conf/init/notificationworker/run diff --git a/Dockerfile.web b/Dockerfile.web index 6cfd76f94..b5bb5e460 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -36,6 +36,9 @@ ADD conf/init/runmigration.sh /etc/my_init.d/ ADD conf/init/gunicorn /etc/service/gunicorn ADD conf/init/nginx /etc/service/nginx ADD conf/init/diffsworker /etc/service/diffsworker +ADD conf/init/notificationworker /etc/service/notificationworker + +# TODO: Remove this after the prod CL push ADD conf/init/webhookworker /etc/service/webhookworker # Download any external libs. diff --git a/app.py b/app.py index 5d59ebef6..5046dcb16 100644 --- a/app.py +++ b/app.py @@ -80,6 +80,9 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf reporter=queue_metrics.report) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) +# TODO: Remove this in the prod push following the notifications change. +webhook_queue = WorkQueue(app.config['WEBHOOK_QUEUE_NAME'], tf) + database.configure(app.config) model.config.app_config = app.config model.config.store = storage diff --git a/conf/init/notificationworker/log/run b/conf/init/notificationworker/log/run new file mode 100755 index 000000000..46f8431a7 --- /dev/null +++ b/conf/init/notificationworker/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd -t /var/log/notificationworker/ \ No newline at end of file diff --git a/conf/init/notificationworker/run b/conf/init/notificationworker/run new file mode 100755 index 000000000..b149d9f34 --- /dev/null +++ b/conf/init/notificationworker/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting notification worker' + +cd / +venv/bin/python -m workers.notificationworker + +echo 'Notification worker exited' \ No newline at end of file diff --git a/config.py b/config.py index df3216c0e..c6637a7ef 100644 --- a/config.py +++ b/config.py @@ -125,6 +125,9 @@ class DefaultConfig(object): DIFFS_QUEUE_NAME = 'imagediff' DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild' + # TODO: Remove this in the prod push following the notifications change. + WEBHOOK_QUEUE_NAME = 'webhook' + # Super user config. Note: This MUST BE an empty list for the default config. SUPER_USERS = [] diff --git a/data/model/legacy.py b/data/model/legacy.py index e428f174f..b5afdfeb8 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -56,10 +56,6 @@ class InvalidRepositoryBuildException(DataModelException): pass -class InvalidWebhookException(DataModelException): - pass - - class InvalidNotificationException(DataModelException): pass @@ -1552,13 +1548,6 @@ def create_repo_notification(repo, event_name, method_name, config): config_json=json.dumps(config)) -def lookup_repo_notification(notification_id): - try: - return RepositoryNotification.get(RepositoryNotification.id == notification_id) - except RepositoryNotification.DoesNotExist: - return None - - def get_repo_notification(namespace_name, repository_name, uuid): joined = RepositoryNotification.select().join(Repository) found = list(joined.where(Repository.namespace == namespace_name, @@ -1588,34 +1577,6 @@ def list_repo_notifications(namespace_name, repository_name, event_name=None): return where -# TODO: remove webhook methods when no longer used. -def create_webhook(repo, params_obj): - return Webhook.create(repository=repo, parameters=json.dumps(params_obj)) - - -def get_webhook(namespace_name, repository_name, public_id): - joined = Webhook.select().join(Repository) - found = list(joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name, - Webhook.public_id == public_id)) - - if not found: - raise InvalidWebhookException('No webhook found with id: %s' % public_id) - - return found[0] - - -def list_webhooks(namespace_name, repository_name): - joined = Webhook.select().join(Repository) - return joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name) - - -def delete_webhook(namespace_name, repository_name, public_id): - webhook = get_webhook(namespace_name, repository_name, public_id) - webhook.delete_instance() - return webhook - def list_logs(start_time, end_time, performer=None, repository=None, namespace=None): joined = LogEntry.select().join(User) diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 36764866d..1fab89dd0 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -8,6 +8,7 @@ from endpoints.api import (RepositoryParamResource, nickname, resource, require_ from endpoints.notificationevent import NotificationEvent from endpoints.notificationmethod import (NotificationMethod, CannotValidateNotificationMethodException) +from endpoints.notificationhelper import build_notification_data from data import model @@ -134,11 +135,7 @@ class TestRepositoryNotification(RepositoryParamResource): event_info = NotificationEvent.get_event(notification.event.name) sample_data = event_info.get_sample_data(repository=notification.repository) - notification_data = { - 'notification_id': notification.id, - 'repository_id': notification.repository.id, - 'event_data': sample_data - } + notification_data = build_notification_data(notification, sample_data) notification_queue.put([namespace, repository, notification.event.name], json.dumps(notification_data)) diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py index 0788a1a33..ed0e20716 100644 --- a/endpoints/notificationevent.py +++ b/endpoints/notificationevent.py @@ -71,7 +71,7 @@ class RepoPushEvent(NotificationEvent): Tags Updated: %s """ % (event_data['homepage'], event_data['repository'], - event_data['updated_tags']) + ', '.join(event_data['updated_tags'])) return html diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py index 29d79fb89..773779fb7 100644 --- a/endpoints/notificationhelper.py +++ b/endpoints/notificationhelper.py @@ -27,17 +27,20 @@ def build_event_data(repo, extra_data={}, subpage=None): event_data.update(extra_data) return event_data +def build_notification_data(notification, event_data): + return { + 'notification_uuid': notification.uuid, + 'repository_namespace': notification.repository.namespace, + 'repository_name': notification.repository.name, + 'event_data': event_data + } + def spawn_notification(repo, event_name, extra_data={}, subpage=None, pathargs=[]): event_data = build_event_data(repo, extra_data=extra_data, subpage=subpage) notifications = model.list_repo_notifications(repo.namespace, repo.name, event_name=event_name) for notification in notifications: - notification_data = { - 'notification_id': notification.id, - 'repository_id': repo.id, - 'event_data': event_data - } - + notification_data = build_notification_data(notification, event_data) path = [repo.namespace, repo.name, event_name] + pathargs notification_queue.put(path, json.dumps(notification_data)) diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index b86f0cdd0..b49055157 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -100,8 +100,7 @@ class QuayNotificationMethod(NotificationMethod): def perform(self, notification, event_handler, notification_data): - repository_id = notification_data['repository_id'] - repository = model.lookup_repository(repository_id) + repository = notification.repository if not repository: # Probably deleted. return True diff --git a/workers/notificationworker.py b/workers/notificationworker.py index 17bc514ff..2af0954f9 100644 --- a/workers/notificationworker.py +++ b/workers/notificationworker.py @@ -22,9 +22,11 @@ logger = logging.getLogger(__name__) class NotificationWorker(Worker): def process_queue_item(self, job_details): - notification_id = job_details['notification_id']; - notification = model.lookup_repo_notification(notification_id) - + notification_uuid = job_details['notification_uuid']; + repo_namespace = job_details['repository_namespace'] + repo_name = job_details['repository_name'] + + notification = model.get_repo_notification(repo_namespace, repo_name, notification_uuid) if not notification: # Probably deleted. return True