diff --git a/app.py b/app.py index 8fd6a43ef..ac6f93a96 100644 --- a/app.py +++ b/app.py @@ -71,14 +71,14 @@ sentry = Sentry(app) build_logs = BuildLogs(app) queue_metrics = QueueMetrics(app) authentication = UserAuthentication(app) -expiration = Expiration(app) +#expiration = Expiration(app) userevents = UserEventsBuilderModule(app) tf = app.config['DB_TRANSACTION_FACTORY'] image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, reporter=queue_metrics.report) -webhook_queue = WorkQueue(app.config['WEBHOOK_QUEUE_NAME'], tf) +notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) database.configure(app.config) model.config.app_config = app.config diff --git a/config.py b/config.py index 79866aaf2..df3216c0e 100644 --- a/config.py +++ b/config.py @@ -121,7 +121,7 @@ class DefaultConfig(object): with open(tag_path) as tag_svg: STATUS_TAGS[tag_name] = tag_svg.read() - WEBHOOK_QUEUE_NAME = 'webhook' + NOTIFICATION_QUEUE_NAME = 'notification' DIFFS_QUEUE_NAME = 'imagediff' DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild' diff --git a/data/database.py b/data/database.py index e8fd7c8c3..a1f34fe52 100644 --- a/data/database.py +++ b/data/database.py @@ -382,18 +382,10 @@ class RepositoryNotification(BaseModel): config_json = TextField() -# TODO: remove after migration. -class Webhook(BaseModel): - public_id = CharField(default=random_string_generator(length=64), - unique=True, index=True) - repository = ForeignKeyField(Repository) - parameters = TextField() - - all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem, RepositoryBuild, Team, TeamMember, TeamRole, LogEntryKind, LogEntry, PermissionPrototype, ImageStorage, BuildTriggerService, RepositoryBuildTrigger, OAuthApplication, OAuthAuthorizationCode, OAuthAccessToken, NotificationKind, Notification, ImageStorageLocation, ImageStoragePlacement, - ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, Webhook] + ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification] diff --git a/data/model/legacy.py b/data/model/legacy.py index 860b0af87..7a6df3cb1 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -840,6 +840,13 @@ def get_repository_for_resource(resource_key): return None +def lookup_repository(repo_id): + try: + return Repository.get(Repository.id == repo_id) + except Repository.DoesNotExist: + return None + + def get_repository(namespace_name, repository_name): try: return Repository.get(Repository.name == repository_name, @@ -1540,6 +1547,13 @@ def create_repo_notification(repo, event_name, method_name, config): config_json=json.dumps(config)) +def lookup_repo_notification(notification_id): + try: + return RepositoryNotification.get(RepositoryNotification.id == notification_id) + except RepositoryNotification.DoesNotExist: + return None + + def get_repo_notification(namespace_name, repository_name, uuid): joined = RepositoryNotification.select().join(Repository) found = list(joined.where(Repository.namespace == namespace_name, @@ -1558,11 +1572,16 @@ def delete_repo_notification(namespace_name, repository_name, uuid): return found -def list_repo_notifications(namespace_name, repository_name): +def list_repo_notifications(namespace_name, repository_name, event_name=None): joined = RepositoryNotification.select().join(Repository) - return joined.where(Repository.namespace == namespace_name, - Repository.name == repository_name) + where = joined.where(Repository.namespace == namespace_name, + Repository.name == repository_name) + if event_name: + event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name) + where = where.where(Repostiory.event == event) + + return where # TODO: remove webhook methods when no longer used. def create_webhook(repo, params_obj): diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 1b34ad38a..775e7328b 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -316,4 +316,3 @@ import endpoints.api.tag import endpoints.api.team import endpoints.api.trigger import endpoints.api.user -import endpoints.api.webhook diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 5315d5282..175f45350 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -2,8 +2,10 @@ import json from flask import request +from app import notification_queue from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin, log_action, validate_json_request, api, NotFound) +from endpoints.notificationevent import NotificationEvent from data import model @@ -23,7 +25,7 @@ def notification_view(notification): @resource('/v1/repository//notification/') -class NotificaitonList(RepositoryParamResource): +class RepositoryNotificationList(RepositoryParamResource): """ Resource for dealing with listing and creating notifications on a repository. """ schemas = { 'NotificationCreateRequest': { @@ -81,7 +83,7 @@ class NotificaitonList(RepositoryParamResource): @resource('/v1/repository//notification/') -class Notification(RepositoryParamResource): +class RepositoryNotification(RepositoryParamResource): """ Resource for dealing with specific notifications. """ @require_repo_admin @nickname('getRepoNotification') @@ -105,3 +107,28 @@ class Notification(RepositoryParamResource): repo=model.get_repository(namespace, repository)) return 'No Content', 204 + + +@resource('/v1/repository//notification//test') +class TestRepositoryNotification(RepositoryParamResource): + """ Resource for queuing a test of a notification. """ + @require_repo_admin + @nickname('testRepoNotification') + def post(self, namespace, repository, uuid): + """ Queues a test notification for this repository. """ + try: + notification = model.get_repo_notification(namespace, repository, uuid) + except model.InvalidNotificationException: + raise NotFound() + + event_info = NotificationEvent.get_event(notification.event.name) + sample_data = event_info.get_sample_data(repository=notification.repository) + notification_data = { + 'notification_id': notification.id, + 'repository_id': notification.repository.id, + 'event_data': sample_data + } + notification_queue.put([namespace, repository, notification.event.name], + json.dumps(notification_data)) + + return {} diff --git a/endpoints/api/webhook.py b/endpoints/api/webhook.py deleted file mode 100644 index b38d7ec43..000000000 --- a/endpoints/api/webhook.py +++ /dev/null @@ -1,77 +0,0 @@ -import json - -from flask import request - -from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin, - log_action, validate_json_request, api, NotFound) -from data import model - - -def webhook_view(webhook): - return { - 'public_id': webhook.public_id, - 'parameters': json.loads(webhook.parameters), - } - - -@resource('/v1/repository//webhook/') -class WebhookList(RepositoryParamResource): - """ Resource for dealing with listing and creating webhooks. """ - schemas = { - 'WebhookCreateRequest': { - 'id': 'WebhookCreateRequest', - 'type': 'object', - 'description': 'Arbitrary json.', - }, - } - - @require_repo_admin - @nickname('createWebhook') - @validate_json_request('WebhookCreateRequest') - def post(self, namespace, repository): - """ Create a new webhook for the specified repository. """ - repo = model.get_repository(namespace, repository) - webhook = model.create_webhook(repo, request.get_json()) - resp = webhook_view(webhook) - repo_string = '%s/%s' % (namespace, repository) - headers = { - 'Location': api.url_for(Webhook, repository=repo_string, public_id=webhook.public_id), - } - log_action('add_repo_webhook', namespace, - {'repo': repository, 'webhook_id': webhook.public_id}, - repo=repo) - return resp, 201, headers - - @require_repo_admin - @nickname('listWebhooks') - def get(self, namespace, repository): - """ List the webhooks for the specified repository. """ - webhooks = model.list_webhooks(namespace, repository) - return { - 'webhooks': [webhook_view(webhook) for webhook in webhooks] - } - - -@resource('/v1/repository//webhook/') -class Webhook(RepositoryParamResource): - """ Resource for dealing with specific webhooks. """ - @require_repo_admin - @nickname('getWebhook') - def get(self, namespace, repository, public_id): - """ Get information for the specified webhook. """ - try: - webhook = model.get_webhook(namespace, repository, public_id) - except model.InvalidWebhookException: - raise NotFound() - - return webhook_view(webhook) - - @require_repo_admin - @nickname('deleteWebhook') - def delete(self, namespace, repository, public_id): - """ Delete the specified webhook. """ - model.delete_webhook(namespace, repository, public_id) - log_action('delete_repo_webhook', namespace, - {'repo': repository, 'webhook_id': public_id}, - repo=model.get_repository(namespace, repository)) - return 'No Content', 204 diff --git a/endpoints/index.py b/endpoints/index.py index 0e063882f..a410f9b38 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -8,7 +8,7 @@ from collections import OrderedDict from data import model from data.model import oauth -from app import analytics, app, webhook_queue, authentication, userevents, storage +from app import analytics, app, notification_queue, authentication, userevents, storage from auth.auth import process_auth from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token from util.names import parse_repository_name @@ -315,27 +315,30 @@ def update_images(namespace, repository): profile.debug('GCing repository') num_removed = model.garbage_collect_repository(namespace, repository) - # Generate a job for each webhook that has been added to this repo - profile.debug('Adding webhooks for repository') + # Generate a job for each notification that has been added to this repo + profile.debug('Adding notifications for repository') - webhooks = model.list_webhooks(namespace, repository) - for webhook in webhooks: - webhook_data = json.loads(webhook.parameters) - repo_string = '%s/%s' % (namespace, repository) - profile.debug('Creating webhook for repository \'%s\' for url \'%s\'', - repo_string, webhook_data['url']) - webhook_data['payload'] = { - 'repository': repo_string, - 'namespace': namespace, - 'name': repository, - 'docker_url': 'quay.io/%s' % repo_string, - 'homepage': 'https://quay.io/repository/%s' % repo_string, - 'visibility': repo.visibility.name, - 'updated_tags': updated_tags, - 'pushed_image_count': len(image_with_checksums), - 'pruned_image_count': num_removed, + repo_string = '%s/%s' % (namespace, repository) + event_data = { + 'repository': repo_string, + 'namespace': namespace, + 'name': repository, + 'docker_url': 'quay.io/%s' % repo_string, + 'homepage': 'https://quay.io/repository/%s' % repo_string, + 'visibility': repo.visibility.name, + 'updated_tags': updated_tags, + 'pushed_image_count': len(image_with_checksums), + 'pruned_image_count': num_removed + } + + notifications = model.list_repo_notifications(namespace, repository, event_name='repo_push') + for notification in notifications: + notification_data = { + 'notification_id': notification.id, + 'repository_id': repository.id, + 'event_data': event_data } - webhook_queue.put([namespace, repository], json.dumps(webhook_data)) + notification_queue.put([namespace, repository, 'repo_push'], json.dumps(notification_data)) return make_response('Updated', 204) diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py new file mode 100644 index 000000000..c091a204c --- /dev/null +++ b/endpoints/notificationevent.py @@ -0,0 +1,108 @@ +import logging +import io +import os.path +import tarfile +import base64 + +logger = logging.getLogger(__name__) + +class InvalidNotificationEventException(Exception): + pass + +class NotificationEvent(object): + def __init__(self): + pass + + def get_summary(self, notification_data): + """ + Returns a human readable one-line summary for the given notification data. + """ + raise NotImplementedError + + def get_message(self, notification_data): + """ + Returns a human readable HTML message for the given notification data. + """ + raise NotImplementedError + + def get_sample_data(self, repository=None): + """ + Returns sample data for testing the raising of this notification, with an optional + repository. + """ + raise NotImplementedError + + @classmethod + def event_name(cls): + """ + Particular event implemented by subclasses. + """ + raise NotImplementedError + + @classmethod + def get_event(cls, eventname): + for subc in cls.__subclasses__(): + if subc.event_name() == eventname: + return subc() + + raise InvalidNotificationEventException('Unable to find event: %s' % eventname) + + +class RepoPushEvent(NotificationEvent): + @classmethod + def event_name(cls): + return 'repo_push' + + def get_summary(self, notification_data): + return 'Repository %s updated' % (event_data['repository']) + + def get_message(self, notification_data): + event_data = notification_data['event_data'] + if not event_data['tags']: + return '%s images pushed for repository %s (%s)' % (event_data['pushed_image_count'], + event_data['repository'], event_data['homepage']) + + return 'Tags %s updated for repository %s (%s)' % (event_data['updated_tags'], + event_data['repository'], event_data['homepage']) + + def get_sample_data(self, repository=None): + repo_string = '%s/%s' % (repository.namespace, repository.name) + event_data = { + 'repository': repo_string, + 'namespace': repository.namespace, + 'name': repository.name, + 'docker_url': 'quay.io/%s' % repo_string, + 'homepage': 'https://quay.io/repository/%s' % repo_string, + 'visibility': repository.visibility.name, + 'updated_tags': ['latest', 'foo', 'bar'], + 'pushed_image_count': 10, + 'pruned_image_count': 3 + } + return event_data + + +class BuildStartEvent(NotificationEvent): + @classmethod + def event_name(cls): + return 'build_start' + + def get_sample_data(repository=None): + pass + + +class BuildSuccessEvent(NotificationEvent): + @classmethod + def event_name(cls): + return 'build_success' + + def get_sample_data(repository=None): + pass + + +class BuildFailureEvent(NotificationEvent): + @classmethod + def event_name(cls): + return 'build_failure' + + def get_sample_data(repository=None): + pass diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py new file mode 100644 index 000000000..011a6b010 --- /dev/null +++ b/endpoints/notificationmethod.py @@ -0,0 +1,114 @@ +import logging +import io +import os.path +import tarfile +import base64 +import json + +from flask.ext.mail import Message +from app import mail, app +from data import model + +logger = logging.getLogger(__name__) + +class InvalidNotificationMethodException(Exception): + pass + +class NotificationMethod(object): + def __init__(self): + pass + + @classmethod + def method_name(cls): + """ + Particular method implemented by subclasses. + """ + raise NotImplementedError + + def perform(self, notification, event_handler, notification_data): + """ + Performs the notification method. + + notification: The noticication record itself. + event_handler: The NotificationEvent handler. + notification_data: The dict of notification data placed in the queue. + """ + raise NotImplementedError + + @classmethod + def get_method(cls, methodname): + for subc in cls.__subclasses__(): + if subc.method_name() == methodname: + return subc() + + raise InvalidNotificationMethodException('Unable to find method: %s' % methodname) + + +class QuayNotificationMethod(NotificationMethod): + @classmethod + def method_name(cls): + return 'quay_notification' + + def perform(self, notification, event_handler, notification_data): + repository_id = notification_data['repository_id'] + repository = model.lookup_repository(repository_id) + if not repository: + # Probably deleted. + return True + + model.create_notification(event_handler.event_name(), + repository.namespace, metadata=notification_data['event_data']) + return True + + +class EmailMethod(NotificationMethod): + @classmethod + def method_name(cls): + return 'email' + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + email = config_data.get('email', '') + if not email: + return False + + msg = Message(event_handler.get_summary(notification_data), + sender='support@quay.io', + recipients=[email]) + msg.html = event_handler.get_message(notification_data) + + try: + mail.send(msg) + except Exception as ex: + logger.exception('Email was unable to be sent: %s' % ex.message) + return False + + return True + + +class WebhookMethod(NotificationMethod): + @classmethod + def method_name(cls): + return 'webhook' + + def perform(self, notification, event_handler, notification_data): + config_data = json.loads(notification.config_json) + url = config_data.get('url', '') + if not url: + return False + + payload = notification_data['event_data'] + headers = {'Content-type': 'application/json'} + + try: + resp = requests.post(url, data=json.dumps(payload), headers=headers) + if resp.status_code/100 != 2: + logger.error('%s response for webhook to url: %s' % (resp.status_code, + url)) + return False + + except requests.exceptions.RequestException as ex: + logger.exception('Webhook was unable to be sent: %s' % ex.message) + return False + + return True diff --git a/initdb.py b/initdb.py index 50c6f6197..20d442c8f 100644 --- a/initdb.py +++ b/initdb.py @@ -229,23 +229,18 @@ def initialize_database(): LogEntryKind.create(name='delete_application') LogEntryKind.create(name='reset_application_client_secret') - # TODO: remove these when webhooks are removed. + # Note: These are deprecated. LogEntryKind.create(name='add_repo_webhook') LogEntryKind.create(name='delete_repo_webhook') LogEntryKind.create(name='add_repo_notification') LogEntryKind.create(name='delete_repo_notification') - NotificationKind.create(name='password_required') - NotificationKind.create(name='over_private_usage') - NotificationKind.create(name='expiring_license') - NotificationKind.create(name='maintenance') - - NotificationKind.create(name='test_notification') - ImageStorageLocation.create(name='local_eu') ImageStorageLocation.create(name='local_us') + # NOTE: These MUST be copied over to NotificationKind, since every external + # notification can also generate a Quay.io notification. ExternalNotificationEvent.create(name='repo_push') ExternalNotificationEvent.create(name='build_start') ExternalNotificationEvent.create(name='build_success') @@ -255,6 +250,18 @@ def initialize_database(): ExternalNotificationMethod.create(name='email') ExternalNotificationMethod.create(name='webhook') + NotificationKind.create(name='repo_push') + NotificationKind.create(name='build_start') + NotificationKind.create(name='build_success') + NotificationKind.create(name='build_failure') + + NotificationKind.create(name='password_required') + NotificationKind.create(name='over_private_usage') + NotificationKind.create(name='expiring_license') + NotificationKind.create(name='maintenance') + + NotificationKind.create(name='test_notification') + def wipe_database(): logger.debug('Wiping all data from the DB.') diff --git a/license.pyc b/license.pyc index df6085268..83687adfa 100644 Binary files a/license.pyc and b/license.pyc differ diff --git a/static/js/app.js b/static/js/app.js index 9f1cc4456..ff3275b2d 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -2362,7 +2362,8 @@ quayApp.directive('logsView', function () { 'repository': '=repository', 'performer': '=performer' }, - controller: function($scope, $element, $sce, Restangular, ApiService, TriggerDescriptionBuilder, StringBuilderService) { + controller: function($scope, $element, $sce, Restangular, ApiService, TriggerDescriptionBuilder, + StringBuilderService, ExternalNotificationData) { $scope.loading = true; $scope.logs = null; $scope.kindsAllowed = null; @@ -2423,8 +2424,6 @@ quayApp.directive('logsView', function () { 'change_repo_visibility': 'Change visibility for repository {repo} to {visibility}', 'add_repo_accesstoken': 'Create access token {token} in repository {repo}', 'delete_repo_accesstoken': 'Delete access token {token} in repository {repo}', - 'add_repo_webhook': 'Add webhook in repository {repo}', - 'delete_repo_webhook': 'Delete webhook in repository {repo}', 'set_repo_description': 'Change description for repository {repo}: {description}', 'build_dockerfile': function(metadata) { if (metadata.trigger_id) { @@ -2475,7 +2474,21 @@ quayApp.directive('logsView', function () { 'update_application': 'Update application to {application_name} for client ID {client_id}', 'delete_application': 'Delete application {application_name} with client ID {client_id}', 'reset_application_client_secret': 'Reset the Client Secret of application {application_name} ' + - 'with client ID {client_id}' + 'with client ID {client_id}', + + 'add_repo_notification': function(metadata) { + var eventData = ExternalNotificationData.getEventInfo(metadata.event); + return 'Add notification of event "' + eventData['title'] + '" for repository {repo}'; + }, + + 'delete_repo_notification': function(metadata) { + var eventData = ExternalNotificationData.getEventInfo(metadata.event); + return 'Delete notification of event "' + eventData['title'] + '" for repository {repo}'; + }, + + // Note: These are deprecated. + 'add_repo_webhook': 'Add webhook in repository {repo}', + 'delete_repo_webhook': 'Delete webhook in repository {repo}' }; var logKinds = { @@ -2494,8 +2507,6 @@ quayApp.directive('logsView', function () { 'change_repo_visibility': 'Change repository visibility', 'add_repo_accesstoken': 'Create access token', 'delete_repo_accesstoken': 'Delete access token', - 'add_repo_webhook': 'Add webhook', - 'delete_repo_webhook': 'Delete webhook', 'set_repo_description': 'Change repository description', 'build_dockerfile': 'Build image from Dockerfile', 'delete_tag': 'Delete Tag', @@ -2515,7 +2526,13 @@ quayApp.directive('logsView', function () { 'create_application': 'Create Application', 'update_application': 'Update Application', 'delete_application': 'Delete Application', - 'reset_application_client_secret': 'Reset Client Secret' + 'reset_application_client_secret': 'Reset Client Secret', + 'add_repo_notification': 'Add repository notification', + 'delete_repo_notification': 'Delete repository notification', + + // Note: these are deprecated. + 'add_repo_webhook': 'Add webhook', + 'delete_repo_webhook': 'Delete webhook' }; var getDateString = function(date) { diff --git a/test/data/test.db b/test/data/test.db index 8ec8dfa41..29744f9e4 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_security.py b/test/test_api_security.py index 34fe7ee18..b8e8470c5 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -18,7 +18,7 @@ from endpoints.api.robot import UserRobotList, OrgRobot, OrgRobotList, UserRobot from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, TriggerBuildList, ActivateBuildTrigger, BuildTrigger, BuildTriggerList, BuildTriggerAnalyze) -from endpoints.api.webhook import Webhook, WebhookList +from endpoints.api.repositorynotification import RepositoryNotification, RepositoryNotificationList from endpoints.api.user import (PrivateRepositories, ConvertToOrganization, Recovery, Signout, Signin, User, UserAuthorizationList, UserAuthorization) from endpoints.api.repotoken import RepositoryToken, RepositoryTokenList @@ -1883,10 +1883,10 @@ class TestBuildTriggerD6tiBuynlargeOrgrepo(ApiTestCase): self._run_test('DELETE', 404, 'devtable', None) -class TestWebhookQfatPublicPublicrepo(ApiTestCase): +class TestRepositoryNotificationQfatPublicPublicrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(Webhook, public_id="QFAT", repository="public/publicrepo") + self._set_url(RepositoryNotification, uuid="QFAT", repository="public/publicrepo") def test_get_anonymous(self): self._run_test('GET', 401, None, None) @@ -1913,10 +1913,10 @@ class TestWebhookQfatPublicPublicrepo(ApiTestCase): self._run_test('DELETE', 403, 'devtable', None) -class TestWebhookQfatDevtableShared(ApiTestCase): +class TestRepositoryNotificationQfatDevtableShared(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(Webhook, public_id="QFAT", repository="devtable/shared") + self._set_url(RepositoryNotification, uuid="QFAT", repository="devtable/shared") def test_get_anonymous(self): self._run_test('GET', 401, None, None) @@ -1943,10 +1943,10 @@ class TestWebhookQfatDevtableShared(ApiTestCase): self._run_test('DELETE', 400, 'devtable', None) -class TestWebhookQfatBuynlargeOrgrepo(ApiTestCase): +class TestRepositoryNotificationQfatBuynlargeOrgrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(Webhook, public_id="QFAT", repository="buynlarge/orgrepo") + self._set_url(RepositoryNotification, uuid="QFAT", repository="buynlarge/orgrepo") def test_get_anonymous(self): self._run_test('GET', 401, None, None) @@ -2529,10 +2529,10 @@ class TestBuildTriggerListBuynlargeOrgrepo(ApiTestCase): self._run_test('GET', 200, 'devtable', None) -class TestWebhookListPublicPublicrepo(ApiTestCase): +class TestRepositoryNotificationListPublicPublicrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(WebhookList, repository="public/publicrepo") + self._set_url(RepositoryNotificationList, repository="public/publicrepo") def test_get_anonymous(self): self._run_test('GET', 401, None, None) @@ -2559,10 +2559,10 @@ class TestWebhookListPublicPublicrepo(ApiTestCase): self._run_test('POST', 403, 'devtable', {}) -class TestWebhookListDevtableShared(ApiTestCase): +class TestRepositoryNotificationListDevtableShared(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(WebhookList, repository="devtable/shared") + self._set_url(RepositoryNotificationList, repository="devtable/shared") def test_get_anonymous(self): self._run_test('GET', 401, None, None) @@ -2586,13 +2586,13 @@ class TestWebhookListDevtableShared(ApiTestCase): self._run_test('POST', 403, 'reader', {}) def test_post_devtable(self): - self._run_test('POST', 201, 'devtable', {}) + self._run_test('POST', 201, 'devtable', {'event': 'repo_push', 'method': 'email', 'config': {}}) -class TestWebhookListBuynlargeOrgrepo(ApiTestCase): +class TestRepositoryNotificationListBuynlargeOrgrepo(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) - self._set_url(WebhookList, repository="buynlarge/orgrepo") + self._set_url(RepositoryNotificationList, repository="buynlarge/orgrepo") def test_get_anonymous(self): self._run_test('GET', 401, None, None) @@ -2616,7 +2616,7 @@ class TestWebhookListBuynlargeOrgrepo(ApiTestCase): self._run_test('POST', 403, 'reader', {}) def test_post_devtable(self): - self._run_test('POST', 201, 'devtable', {}) + self._run_test('POST', 201, 'devtable', {'event': 'repo_push', 'method': 'email', 'config': {}}) class TestRepositoryTokenListPublicPublicrepo(ApiTestCase): diff --git a/test/test_api_usage.py b/test/test_api_usage.py index a2d062bea..e0c21f269 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -20,7 +20,7 @@ from endpoints.api.robot import UserRobotList, OrgRobot, OrgRobotList, UserRobot from endpoints.api.trigger import (BuildTriggerActivate, BuildTriggerSources, BuildTriggerSubdirs, TriggerBuildList, ActivateBuildTrigger, BuildTrigger, BuildTriggerList, BuildTriggerAnalyze) -from endpoints.api.webhook import Webhook, WebhookList +from endpoints.api.repositorynotification import RepositoryNotification, RepositoryNotificationList from endpoints.api.user import (PrivateRepositories, ConvertToOrganization, Signout, Signin, User, UserAuthorizationList, UserAuthorization) @@ -1073,41 +1073,44 @@ class TestRequestRepoBuild(ApiTestCase): -class TestWebhooks(ApiTestCase): +class TestRepositoryNotifications(ApiTestCase): def test_webhooks(self): self.login(ADMIN_ACCESS_USER) - # Add a webhook. - json = self.postJsonResponse(WebhookList, + # Add a notification. + json = self.postJsonResponse(RepositoryNotificationList, params=dict(repository=ADMIN_ACCESS_USER + '/simple'), - data=dict(url='http://example.com'), + data=dict(config={'url': 'http://example.com'}, event='repo_push', method='webhook'), expected_code=201) - self.assertEquals('http://example.com', json['parameters']['url']) - wid = json['public_id'] + self.assertEquals('repo_push', json['event']) + self.assertEquals('webhook', json['method']) + self.assertEquals('http://example.com', json['config']['url']) + wid = json['uuid'] - # Get the webhook. - json = self.getJsonResponse(Webhook, - params=dict(repository=ADMIN_ACCESS_USER + '/simple', public_id=wid)) + # Get the notification. + json = self.getJsonResponse(RepositoryNotification, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', uuid=wid)) - self.assertEquals(wid, json['public_id']) - self.assertEquals('http://example.com', json['parameters']['url']) + self.assertEquals(wid, json['uuid']) + self.assertEquals('repo_push', json['event']) + self.assertEquals('webhook', json['method']) - # Verify the webhook is listed. - json = self.getJsonResponse(WebhookList, + # Verify the notification is listed. + json = self.getJsonResponse(RepositoryNotificationList, params=dict(repository=ADMIN_ACCESS_USER + '/simple')) - ids = [w['public_id'] for w in json['webhooks']] + ids = [w['uuid'] for w in json['notifications']] assert wid in ids - # Delete the webhook. - self.deleteResponse(Webhook, - params=dict(repository=ADMIN_ACCESS_USER + '/simple', public_id=wid), + # Delete the notification. + self.deleteResponse(RepositoryNotification, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', uuid=wid), expected_code=204) - # Verify the webhook is gone. - self.getResponse(Webhook, - params=dict(repository=ADMIN_ACCESS_USER + '/simple', public_id=wid), + # Verify the notification is gone. + self.getResponse(RepositoryNotification, + params=dict(repository=ADMIN_ACCESS_USER + '/simple', uuid=wid), expected_code=404) diff --git a/workers/notificationworker.py b/workers/notificationworker.py new file mode 100644 index 000000000..4ccee7a2d --- /dev/null +++ b/workers/notificationworker.py @@ -0,0 +1,54 @@ +import logging +import argparse +import requests +import json + +from app import notification_queue +from workers.worker import Worker + +from endpoints.notificationmethod import NotificationMethod, InvalidNotificationMethodException +from endpoints.notificationevent import NotificationEvent, InvalidNotificationEventException + +from data import model + +root_logger = logging.getLogger('') +root_logger.setLevel(logging.DEBUG) + +FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' +formatter = logging.Formatter(FORMAT) + +logger = logging.getLogger(__name__) + + +class NotificationWorker(Worker): + def process_queue_item(self, job_details): + notification_id = job_details['notification_id']; + notification = model.lookup_repo_notification(notification_id) + + print job_details + print notification + + if not notification: + # Probably deleted. + return True + + event_name = notification.event.name + method_name = notification.method.name + + try: + event_handler = NotificationEvent.get_event(event_name) + method_handler = NotificationMethod.get_method(method_name) + except InvalidNotificationMethodException as ex: + logger.exception('Cannot find notification method: %s' % ex.message) + return False + except InvalidNotificationEventException as ex: + logger.exception('Cannot find notification method: %s' % ex.message) + return False + + return method_handler.perform(notification, event_handler, job_details) + +logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) + +worker = NotificationWorker(notification_queue, poll_period_seconds=15, + reservation_seconds=3600) +worker.start()