diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 655c9f6e3..194178578 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -1,19 +1,15 @@ import logging import datetime -import json from calendar import timegm from email.utils import formatdate from functools import partial, wraps -from enum import Enum -from flask import Blueprint, Response, request, make_response, jsonify, session, url_for +from flask import Blueprint, request, session from flask_restful import Resource, abort, Api, reqparse from flask_restful.utils.cors import crossdomain from jsonschema import validate, ValidationError -import features - from app import app, metric_queue from data import model from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, @@ -23,16 +19,13 @@ from auth import scopes from auth.auth_context import get_authenticated_user, get_validated_oauth_token from auth.decorators import process_oauth from endpoints.csrf import csrf_protect -from endpoints.exception import (ApiException, Unauthorized, InvalidRequest, InvalidResponse, +from endpoints.exception import (Unauthorized, InvalidRequest, InvalidResponse, FreshLoginRequired, NotFound) from endpoints.decorators import check_anon_protection -from endpoints.decorated import (handle_dme, handle_emailexception, handle_configexception, - handle_too_many_login_attempts) -from util.config.provider.baseprovider import CannotWriteConfigException from util.metrics.metricqueue import time_decorator from util.names import parse_namespace_repository from util.pagination import encrypt_page_token, decrypt_page_token -from util.useremails import CannotSendEmailException + logger = logging.getLogger(__name__) api_bp = Blueprint('api', __name__) @@ -356,7 +349,12 @@ def request_error(exception=None, **kwargs): def log_action(kind, user_or_orgname, metadata=None, repo=None, repo_name=None): if not metadata: metadata = {} - + + if repo_name: + repository = model.repository.get_repository(user_or_orgname, repo_name) + else: + repository = repo + oauth_token = get_validated_oauth_token() if oauth_token: metadata['oauth_token_id'] = oauth_token.id diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 6ff5bb7f2..10192479b 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -1,47 +1,20 @@ """ List, create and manage repository events/notifications. """ -import json - import logging from flask import request -from app import notification_queue from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin, log_action, validate_json_request, request_error, path_param, disallow_for_app_repositories) -from endpoints.exception import NotFound -from endpoints.notificationevent import NotificationEvent +from endpoints.exception import NotFound, InvalidRequest from endpoints.notificationmethod import (NotificationMethod, CannotValidateNotificationMethodException) from endpoints.notificationhelper import build_notification_data -from data import model from workers.notificationworker.models_pre_oci import notification +from endpoints.api.repositorynotification_models_pre_oci import pre_oci_model as model logger = logging.getLogger(__name__) -def notification_view(note): - config = {} - try: - config = json.loads(note.config_json) - except: - config = {} - - event_config = {} - try: - event_config = json.loads(note.event_config_json) - except: - event_config = {} - - return { - 'uuid': note.uuid, - 'event': note.event.name, - 'method': note.method.name, - 'config': config, - 'title': note.title, - 'event_config': event_config, - 'number_of_failures': note.number_of_failures, - } - @resource('/v1/repository//notification/') @path_param('repository', 'The full path of the repository. e.g. namespace/name') @@ -86,41 +59,37 @@ class RepositoryNotificationList(RepositoryParamResource): @nickname('createRepoNotification') @disallow_for_app_repositories @validate_json_request('NotificationCreateRequest') - def post(self, namespace, repository): - """ Create a new notification for the specified repository. """ - repo = model.repository.get_repository(namespace, repository) + def post(self, namespace_name, repository_name): parsed = request.get_json() - + method_handler = NotificationMethod.get_method(parsed['method']) - if not method_handler: - raise request_error(message='Unknown method') - try: - method_handler.validate(repo, parsed['config']) + method_handler.validate(namespace_name, repository_name, parsed['config']) except CannotValidateNotificationMethodException as ex: raise request_error(message=ex.message) + + new_notification = model.create_repo_notification(namespace_name, repository_name, + parsed['event'], + parsed['method'], + parsed['config'], + parsed['eventConfig'], + parsed.get('title')) - new_notification = model.notification.create_repo_notification(repo, parsed['event'], - parsed['method'], parsed['config'], - parsed['eventConfig'], - parsed.get('title', None)) - - resp = notification_view(new_notification) - log_action('add_repo_notification', namespace, - {'repo': repository, 'namespace': namespace, + log_action('add_repo_notification', namespace_name, + {'repo': repository_name, 'namespace': namespace_name, 'notification_id': new_notification.uuid, - 'event': parsed['event'], 'method': parsed['method']}, - repo=repo) - return resp, 201 + 'event': new_notification.event_name, 'method': new_notification.method_name}, + repo_name=repository_name) + return new_notification.to_dict(), 201 @require_repo_admin @nickname('listRepoNotifications') @disallow_for_app_repositories - def get(self, namespace, repository): + def get(self, namespace_name, repository_name): """ List the notifications for the specified repository. """ - notifications = model.notification.list_repo_notifications(namespace, repository) + notifications = model.list_repo_notifications(namespace_name, repository_name) return { - 'notifications': [notification_view(n) for n in notifications] + 'notifications': [n.to_dict() for n in notifications] } @@ -132,43 +101,42 @@ class RepositoryNotification(RepositoryParamResource): @require_repo_admin @nickname('getRepoNotification') @disallow_for_app_repositories - def get(self, namespace, repository, uuid): + def get(self, namespace_name, repository_name, uuid): """ Get information for the specified notification. """ - try: - found = model.notification.get_repo_notification(uuid) - except model.InvalidNotificationException: + found = model.get_repo_notification(uuid) + if not found: raise NotFound() - - if (found.repository.namespace_user.username != namespace or - found.repository.name != repository): - raise NotFound() - - return notification_view(found) + return found.to_dict() @require_repo_admin @nickname('deleteRepoNotification') @disallow_for_app_repositories - def delete(self, namespace, repository, uuid): + def delete(self, namespace_name, repository_name, uuid): """ Deletes the specified notification. """ - deleted = model.notification.delete_repo_notification(namespace, repository, uuid) - log_action('delete_repo_notification', namespace, - {'repo': repository, 'namespace': namespace, 'notification_id': uuid, - 'event': deleted.event.name, 'method': deleted.method.name}, - repo=model.repository.get_repository(namespace, repository)) + deleted = model.delete_repo_notification(namespace_name, repository_name, uuid) + if not deleted: + raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid)) + + log_action('delete_repo_notification', namespace_name, + {'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid, + 'event': deleted.event_name, 'method': deleted.method_name}, + repo_name=repository_name) return 'No Content', 204 @require_repo_admin @nickname('resetRepositoryNotificationFailures') @disallow_for_app_repositories - def post(self, namespace, repository, uuid): + def post(self, namespace_name, repository_name, uuid): """ Resets repository notification to 0 failures. """ - reset = model.notification.reset_notification_number_of_failures(namespace, repository, uuid) - if reset is not None: - log_action('reset_repo_notification', namespace, - {'repo': repository, 'namespace': namespace, 'notification_id': uuid, - 'event': reset.event.name, 'method': reset.method.name}, - repo=model.repository.get_repository(namespace, repository)) + reset = model.reset_notification_number_of_failures(namespace_name, repository_name, uuid) + if not reset: + raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid)) + + log_action('reset_repo_notification', namespace_name, + {'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid, + 'event': reset.event_name, 'method': reset.method_name}, + repo_name=repository_name) return 'No Content', 204 @@ -181,25 +149,11 @@ class TestRepositoryNotification(RepositoryParamResource): @require_repo_admin @nickname('testRepoNotification') @disallow_for_app_repositories - def post(self, namespace, repository, uuid): + def post(self, namespace_name, repository_name, uuid): """ Queues a test notification for this repository. """ - try: - test_note = model.notification.get_repo_notification(uuid) - except model.InvalidNotificationException: - raise NotFound() + test_note = model.queue_test_notification(uuid) - if (test_note.repository.namespace_user.username != namespace or - test_note.repository.name != repository): - raise NotFound() - - event_info = NotificationEvent.get_event(test_note.event.name) - - # TODO(jschorr): Stop depending on the worker module's data interface and instead only depend - # on the notification's data interface (to be added). - sample_data = event_info.get_sample_data(notification(test_note)) - - notification_data = build_notification_data(test_note, sample_data) - notification_queue.put([test_note.repository.namespace_user.username, repository, - test_note.event.name], json.dumps(notification_data)) - - return {} + if not test_note: + raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid)) + + return {}, 200 diff --git a/endpoints/api/repositorynotification_models_interface.py b/endpoints/api/repositorynotification_models_interface.py new file mode 100644 index 000000000..4d8154735 --- /dev/null +++ b/endpoints/api/repositorynotification_models_interface.py @@ -0,0 +1,144 @@ +import json + +from abc import ABCMeta, abstractmethod +from collections import namedtuple + +from six import add_metaclass + + +class RepositoryNotification( + namedtuple('RepositoryNotification', [ + 'uuid', + 'title', + 'event_name', + 'method_name', + 'config_json', + 'event_config_json', + 'number_of_failures', + ])): + """ + RepositoryNotification represents a notification for a repository. + :type uuid: string + :type event: string + :type method: string + :type config: string + :type title: string + :type event_config: string + :type number_of_failures: int + """ + def to_dict(self): + try: + config = json.loads(self.config_json) + except ValueError: + config = {} + + try: + event_config = json.loads(self.event_config_json) + except ValueError: + event_config = {} + + return { + 'uuid': self.uuid, + 'title': self.title, + 'event': self.event_name, + 'method': self.method_name, + 'config': config, + 'event_config': event_config, + 'number_of_failures': self.number_of_failures, + } + + +@add_metaclass(ABCMeta) +class RepoNotificationInterface(object): + """ + Interface that represents all data store interactions required by the RepositoryNotification API + """ + + @abstractmethod + def create_repo_notification(self, namespace_name, repository_name, event_name, method_name, method_config, event_config, title=None): + """ + + Args: + namespace_name: namespace of repository + repository_name: name of repository + event_name: name of event + method_name: name of method + method_config: method config, json string + event_config: event config, json string + title: title of the notification + + Returns: + RepositoryNotification object + + """ + pass + + @abstractmethod + def list_repo_notifications(self, namespace_name, repository_name, event_name=None): + """ + + Args: + namespace_name: namespace of repository + repository_name: name of repository + event_name: name of event + + Returns: + list(RepositoryNotification) + """ + pass + + @abstractmethod + def get_repo_notification(self, uuid): + """ + + Args: + uuid: uuid of notification + + Returns: + RepositoryNotification or None + + """ + pass + + @abstractmethod + def delete_repo_notification(self, namespace_name, repository_name, uuid): + """ + + Args: + namespace_name: namespace of repository + repository_name: name of repository + uuid: uuid of notification + + Returns: + RepositoryNotification or None + + """ + pass + + @abstractmethod + def reset_notification_number_of_failures(self, namespace_name, repository_name, uuid): + """ + + Args: + namespace_name: namespace of repository + repository_name: name of repository + uuid: uuid of notification + + Returns: + RepositoryNotification + + """ + pass + + @abstractmethod + def queue_test_notification(self, uuid): + """ + + Args: + uuid: uuid of notification + + Returns: + RepositoryNotification or None + + """ + pass diff --git a/endpoints/api/repositorynotification_models_pre_oci.py b/endpoints/api/repositorynotification_models_pre_oci.py new file mode 100644 index 000000000..e1d4c37e9 --- /dev/null +++ b/endpoints/api/repositorynotification_models_pre_oci.py @@ -0,0 +1,71 @@ +import json + +from app import notification_queue +from data import model +from data.model import InvalidNotificationException +from endpoints.api.repositorynotification_models_interface import RepoNotificationInterface, RepositoryNotification +from endpoints.notificationevent import NotificationEvent +from endpoints.notificationhelper import build_notification_data + + +class RepoNotificationPreOCIModel(RepoNotificationInterface): + + def create_repo_notification(self, namespace_name, repository_name, event_name, method_name, method_config, event_config, title=None): + repository = model.repository.get_repository(namespace_name, repository_name) + return self._notification(model.notification.create_repo_notification(repository, + event_name, + method_name, + method_config, + event_config, + title)) + + def list_repo_notifications(self, namespace_name, repository_name, event_name=None): + return [self._notification(n) + for n in model.notification.list_repo_notifications(namespace_name, repository_name, event_name)] + + def get_repo_notification(self, uuid): + try: + found = model.notification.get_repo_notification(uuid) + except InvalidNotificationException: + return None + return self._notification(found) + + def delete_repo_notification(self, namespace_name, repository_name, uuid): + try: + found = model.notification.delete_repo_notification(namespace_name, repository_name, uuid) + except InvalidNotificationException: + return None + return self._notification(found) + + def reset_notification_number_of_failures(self, namespace_name, repository_name, uuid): + return self._notification( + model.notification.reset_notification_number_of_failures(namespace_name, repository_name, uuid)) + + def queue_test_notification(self, uuid): + try: + notification = model.notification.get_repo_notification(uuid) + except InvalidNotificationException: + return None + + event_info = NotificationEvent.get_event(notification.event.name) + sample_data = event_info.get_sample_data(notification) + notification_data = build_notification_data(notification, sample_data) + notification_queue.put([notification.repository.namespace_user.username, notification.uuid, + notification.event.name], json.dumps(notification_data)) + return self._notification(notification) + + + def _notification(self, notification): + if not notification: + return None + return RepositoryNotification(uuid=notification.uuid, + title=notification.title, + event_name=notification.event.name, + method_name=notification.method.name, + config_json=notification.config_json, + event_config_json=notification.event_config_json, + number_of_failures=notification.number_of_failures) + + + +pre_oci_model = RepoNotificationPreOCIModel() \ No newline at end of file diff --git a/endpoints/api/test/test_repositorynotification.py b/endpoints/api/test/test_repositorynotification.py new file mode 100644 index 000000000..ceccf5a19 --- /dev/null +++ b/endpoints/api/test/test_repositorynotification.py @@ -0,0 +1,84 @@ +import pytest + +from mock import Mock, MagicMock + +from endpoints.api.test.shared import conduct_api_call +from endpoints.api.repositorynotification import RepositoryNotificationList, RepositoryNotification, TestRepositoryNotification +from endpoints.test.shared import client_with_identity +import endpoints.api.repositorynotification_models_interface as iface +from test.fixtures import * + +@pytest.fixture() +def authd_client(client): + with client_with_identity('devtable', client) as cl: + yield cl + +def mock_get_notification(uuid): + mock_notification = MagicMock(iface.RepositoryNotification) + if uuid == 'exists': + mock_notification.return_value = iface.RepositoryNotification( + 'exists', + 'title', + 'event_name', + 'method_name', + 'config_json', + 'event_config_json', + 2, + ) + else: + mock_notification.return_value = None + return mock_notification + +@pytest.mark.parametrize('namespace,repository,body,expected_code',[ + ('devtable', 'simple', dict(config={'url': 'http://example.com'}, event='repo_push', + method='webhook', eventConfig={}, title='test'), 201) +]) +def test_create_repo_notification(namespace, repository, body, expected_code, authd_client): + params = {'repository': namespace + '/' + repository} + conduct_api_call(authd_client, RepositoryNotificationList, 'POST', params, body, expected_code=expected_code) + +@pytest.mark.parametrize('namespace,repository,expected_code',[ + ('devtable', 'simple', 200) +]) +def test_list_repo_notifications(namespace, repository, expected_code, authd_client): + params = {'repository': namespace + '/' + repository} + resp = conduct_api_call(authd_client, RepositoryNotificationList, 'GET', params, expected_code=expected_code).json + assert len(resp['notifications']) > 0 + +@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[ + ('devtable', 'simple', 'exists', 200), + ('devtable', 'simple', 'not found', 404), +]) +def test_get_repo_notification(namespace, repository, uuid, expected_code, authd_client, monkeypatch): + monkeypatch.setattr('endpoints.api.repositorynotification.model.get_repo_notification', mock_get_notification(uuid)) + params = {'repository': namespace + '/' + repository, 'uuid': uuid} + conduct_api_call(authd_client, RepositoryNotification, 'GET', params, expected_code=expected_code) + +@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[ + ('devtable', 'simple', 'exists', 204), + ('devtable', 'simple', 'not found', 400), +]) +def test_delete_repo_notification(namespace, repository, uuid, expected_code, authd_client, monkeypatch): + monkeypatch.setattr('endpoints.api.repositorynotification.model.delete_repo_notification', mock_get_notification(uuid)) + params = {'repository': namespace + '/' + repository, 'uuid': uuid} + conduct_api_call(authd_client, RepositoryNotification, 'DELETE', params, expected_code=expected_code) + + +@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[ + ('devtable', 'simple', 'exists', 204), + ('devtable', 'simple', 'not found', 400), +]) +def test_reset_repo_noticiation(namespace, repository, uuid, expected_code, authd_client, monkeypatch): + monkeypatch.setattr('endpoints.api.repositorynotification.model.reset_notification_number_of_failures', mock_get_notification(uuid)) + params = {'repository': namespace + '/' + repository, 'uuid': uuid} + conduct_api_call(authd_client, RepositoryNotification, 'POST', params, expected_code=expected_code) + + +@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[ + ('devtable', 'simple', 'exists', 200), + ('devtable', 'simple', 'not found', 400), +]) +def test_test_repo_notification(namespace, repository, uuid, expected_code, authd_client, monkeypatch): + monkeypatch.setattr('endpoints.api.repositorynotification.model.queue_test_notification', mock_get_notification(uuid)) + params = {'repository': namespace + '/' + repository, 'uuid': uuid} + conduct_api_call(authd_client, TestRepositoryNotification, 'POST', params, expected_code=expected_code) diff --git a/endpoints/api/test/test_security.py b/endpoints/api/test/test_security.py index 68039aed7..a2dee07f2 100644 --- a/endpoints/api/test/test_security.py +++ b/endpoints/api/test/test_security.py @@ -59,7 +59,7 @@ NOTIFICATION_PARAMS = {'namespace': 'devtable', 'repository': 'devtable/simple', (RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, None, 403), (RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'freshuser', 403), (RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'reader', 403), - (RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'devtable', 204), + (RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'devtable', 400), (RepositoryTrust, 'POST', REPO_PARAMS, {'trust_enabled': True}, None, 403), (RepositoryTrust, 'POST', REPO_PARAMS, {'trust_enabled': True}, 'freshuser', 403), diff --git a/endpoints/notificationhelper.py b/endpoints/notificationhelper.py index 37e5bc56b..18e313b8a 100644 --- a/endpoints/notificationhelper.py +++ b/endpoints/notificationhelper.py @@ -5,13 +5,14 @@ from contextlib import contextmanager from app import app, notification_queue from data import model from auth.auth_context import get_authenticated_user, get_validated_oauth_token +from endpoints.notificationmethod import _get_namespace_name_from DEFAULT_BATCH_SIZE = 1000 def build_event_data(repo, extra_data=None, subpage=None): - repo_string = '%s/%s' % (repo.namespace_name, repo.name) + repo_string = '%s/%s' % (_get_namespace_name_from(repo), repo.name) homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], repo_string) @@ -24,7 +25,7 @@ def build_event_data(repo, extra_data=None, subpage=None): event_data = { 'repository': repo_string, - 'namespace': repo.namespace_name, + 'namespace': _get_namespace_name_from(repo), 'name': repo.name, 'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string), 'homepage': homepage, diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index 5a6d8e515..19c6ad9a5 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -56,7 +56,7 @@ class NotificationMethod(object): """ raise NotImplementedError - def validate(self, repository, config_data): + def validate(self, namespace_name, repository_name, config_data): """ Validates that the notification can be created with the given data. Throws a CannotValidateNotificationMethodException on failure. @@ -88,12 +88,12 @@ class QuayNotificationMethod(NotificationMethod): def method_name(cls): return 'quay_notification' - def validate(self, repository, config_data): - status, err_message, target_users = self.find_targets(repository, config_data) + def validate(self, namespace_name, repository_name, config_data): + status, err_message, target_users = self.find_targets(namespace_name, repository_name, config_data) if err_message: raise CannotValidateNotificationMethodException(err_message) - def find_targets(self, repository, config_data): + def find_targets(self, namespace_name, repository_name, config_data): target_info = config_data['target'] if target_info['kind'] == 'user': @@ -134,7 +134,7 @@ class QuayNotificationMethod(NotificationMethod): # Lookup the target user or team to which we'll send the notification. config_data = notification_obj.method_config_dict - status, err_message, target_users = self.find_targets(repository, config_data) + status, err_message, target_users = self.find_targets(_get_namespace_name_from(repository), repository.name, config_data) if not status: raise NotificationMethodPerformException(err_message) @@ -149,12 +149,11 @@ class EmailMethod(NotificationMethod): def method_name(cls): return 'email' - def validate(self, repository, config_data): + def validate(self, namespace_name, repository_name, config_data): email = config_data.get('email', '') if not email: raise CannotValidateNotificationMethodException('Missing e-mail address') - record = model.repository.get_email_authorized_for_repo(_get_namespace_name_from(repository), repository.name, email) if not record or not record.confirmed: @@ -185,7 +184,7 @@ class WebhookMethod(NotificationMethod): def method_name(cls): return 'webhook' - def validate(self, repository, config_data): + def validate(self, namespace_name, repository_name, config_data): url = config_data.get('url', '') if not url: raise CannotValidateNotificationMethodException('Missing webhook URL') @@ -222,7 +221,7 @@ class FlowdockMethod(NotificationMethod): def method_name(cls): return 'flowdock' - def validate(self, repository, config_data): + def validate(self, namespace_name, repository_name, config_data): token = config_data.get('flow_api_token', '') if not token: raise CannotValidateNotificationMethodException('Missing Flowdock API Token') @@ -274,7 +273,7 @@ class HipchatMethod(NotificationMethod): def method_name(cls): return 'hipchat' - def validate(self, repository, config_data): + def validate(self, namespace_name, repository_name, config_data): if not config_data.get('notification_token', ''): raise CannotValidateNotificationMethodException('Missing Hipchat Room Notification Token') @@ -385,7 +384,7 @@ class SlackMethod(NotificationMethod): def method_name(cls): return 'slack' - def validate(self, repository, config_data): + def validate(self, namespace_name, repository_name, config_data): if not config_data.get('url', ''): raise CannotValidateNotificationMethodException('Missing Slack Callback URL')