vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #2749 from ecordell/ecordell/QUAY-645/repo_notification_interface

Browse source 
add data interface and pre oci impelementation for repo notifications
This commit is contained in:
Evan Cordell 2017-07-18 14:58:43 -04:00 committed by GitHub
commit a6ea16abc5
8 changed files with 370 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
endpoints/api/__init__.py
View file

@ -1,19 +1,15 @@
import logging
import datetime
import json
from calendar import timegm
from email.utils import formatdate
from functools import partial, wraps
from enum import Enum
from flask import Blueprint, Response, request, make_response, jsonify, session, url_for
from flask import Blueprint, request, session
from flask_restful import Resource, abort, Api, reqparse
from flask_restful.utils.cors import crossdomain
from jsonschema import validate, ValidationError
import features
from app import app, metric_queue
from data import model
from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
@ -23,16 +19,13 @@ from auth import scopes
from auth.auth_context import get_authenticated_user, get_validated_oauth_token
from auth.decorators import process_oauth
from endpoints.csrf import csrf_protect
from endpoints.exception import (ApiException, Unauthorized, InvalidRequest, InvalidResponse,
from endpoints.exception import (Unauthorized, InvalidRequest, InvalidResponse,
FreshLoginRequired, NotFound)
from endpoints.decorators import check_anon_protection
from endpoints.decorated import (handle_dme, handle_emailexception, handle_configexception,
handle_too_many_login_attempts)
from util.config.provider.baseprovider import CannotWriteConfigException
from util.metrics.metricqueue import time_decorator
from util.names import parse_namespace_repository
from util.pagination import encrypt_page_token, decrypt_page_token
from util.useremails import CannotSendEmailException
logger = logging.getLogger(__name__)
api_bp = Blueprint('api', __name__)
@ -357,6 +350,11 @@ def log_action(kind, user_or_orgname, metadata=None, repo=None, repo_name=None):
if not metadata:
metadata = {}
if repo_name:
repository = model.repository.get_repository(user_or_orgname, repo_name)
else:
repository = repo
oauth_token = get_validated_oauth_token()
if oauth_token:
metadata['oauth_token_id'] = oauth_token.id

134
endpoints/api/repositorynotification.py
View file

@ -1,47 +1,20 @@
""" List, create and manage repository events/notifications. """
import json
import logging
from flask import request
from app import notification_queue
from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin,
log_action, validate_json_request, request_error,
path_param, disallow_for_app_repositories)
from endpoints.exception import NotFound
from endpoints.notificationevent import NotificationEvent
from endpoints.exception import NotFound, InvalidRequest
from endpoints.notificationmethod import (NotificationMethod,
CannotValidateNotificationMethodException)
from endpoints.notificationhelper import build_notification_data
from data import model
from workers.notificationworker.models_pre_oci import notification
from endpoints.api.repositorynotification_models_pre_oci import pre_oci_model as model
logger = logging.getLogger(__name__)
def notification_view(note):
config = {}
try:
config = json.loads(note.config_json)
except:
config = {}
event_config = {}
try:
event_config = json.loads(note.event_config_json)
except:
event_config = {}
return {
'uuid': note.uuid,
'event': note.event.name,
'method': note.method.name,
'config': config,
'title': note.title,
'event_config': event_config,
'number_of_failures': note.number_of_failures,
}
@resource('/v1/repository/<apirepopath:repository>/notification/')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@ -86,41 +59,37 @@ class RepositoryNotificationList(RepositoryParamResource):
@nickname('createRepoNotification')
@disallow_for_app_repositories
@validate_json_request('NotificationCreateRequest')
def post(self, namespace, repository):
""" Create a new notification for the specified repository. """
repo = model.repository.get_repository(namespace, repository)
def post(self, namespace_name, repository_name):
parsed = request.get_json()
method_handler = NotificationMethod.get_method(parsed['method'])
if not method_handler:
raise request_error(message='Unknown method')
try:
method_handler.validate(repo, parsed['config'])
method_handler.validate(namespace_name, repository_name, parsed['config'])
except CannotValidateNotificationMethodException as ex:
raise request_error(message=ex.message)
new_notification = model.notification.create_repo_notification(repo, parsed['event'],
parsed['method'], parsed['config'],
new_notification = model.create_repo_notification(namespace_name, repository_name,
parsed['event'],
parsed['method'],
parsed['config'],
parsed['eventConfig'],
parsed.get('title', None))
parsed.get('title'))
resp = notification_view(new_notification)
log_action('add_repo_notification', namespace,
{'repo': repository, 'namespace': namespace,
log_action('add_repo_notification', namespace_name,
{'repo': repository_name, 'namespace': namespace_name,
'notification_id': new_notification.uuid,
'event': parsed['event'], 'method': parsed['method']},
repo=repo)
return resp, 201
'event': new_notification.event_name, 'method': new_notification.method_name},
repo_name=repository_name)
return new_notification.to_dict(), 201
@require_repo_admin
@nickname('listRepoNotifications')
@disallow_for_app_repositories
def get(self, namespace, repository):
def get(self, namespace_name, repository_name):
""" List the notifications for the specified repository. """
notifications = model.notification.list_repo_notifications(namespace, repository)
notifications = model.list_repo_notifications(namespace_name, repository_name)
return {
'notifications': [notification_view(n) for n in notifications]
'notifications': [n.to_dict() for n in notifications]
}
@ -132,43 +101,42 @@ class RepositoryNotification(RepositoryParamResource):
@require_repo_admin
@nickname('getRepoNotification')
@disallow_for_app_repositories
def get(self, namespace, repository, uuid):
def get(self, namespace_name, repository_name, uuid):
""" Get information for the specified notification. """
try:
found = model.notification.get_repo_notification(uuid)
except model.InvalidNotificationException:
found = model.get_repo_notification(uuid)
if not found:
raise NotFound()
if (found.repository.namespace_user.username != namespace or
found.repository.name != repository):
raise NotFound()
return notification_view(found)
return found.to_dict()
@require_repo_admin
@nickname('deleteRepoNotification')
@disallow_for_app_repositories
def delete(self, namespace, repository, uuid):
def delete(self, namespace_name, repository_name, uuid):
""" Deletes the specified notification. """
deleted = model.notification.delete_repo_notification(namespace, repository, uuid)
log_action('delete_repo_notification', namespace,
{'repo': repository, 'namespace': namespace, 'notification_id': uuid,
'event': deleted.event.name, 'method': deleted.method.name},
repo=model.repository.get_repository(namespace, repository))
deleted = model.delete_repo_notification(namespace_name, repository_name, uuid)
if not deleted:
raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid))
log_action('delete_repo_notification', namespace_name,
{'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid,
'event': deleted.event_name, 'method': deleted.method_name},
repo_name=repository_name)
return 'No Content', 204
@require_repo_admin
@nickname('resetRepositoryNotificationFailures')
@disallow_for_app_repositories
def post(self, namespace, repository, uuid):
def post(self, namespace_name, repository_name, uuid):
""" Resets repository notification to 0 failures. """
reset = model.notification.reset_notification_number_of_failures(namespace, repository, uuid)
if reset is not None:
log_action('reset_repo_notification', namespace,
{'repo': repository, 'namespace': namespace, 'notification_id': uuid,
'event': reset.event.name, 'method': reset.method.name},
repo=model.repository.get_repository(namespace, repository))
reset = model.reset_notification_number_of_failures(namespace_name, repository_name, uuid)
if not reset:
raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid))
log_action('reset_repo_notification', namespace_name,
{'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid,
'event': reset.event_name, 'method': reset.method_name},
repo_name=repository_name)
return 'No Content', 204
@ -181,25 +149,11 @@ class TestRepositoryNotification(RepositoryParamResource):
@require_repo_admin
@nickname('testRepoNotification')
@disallow_for_app_repositories
def post(self, namespace, repository, uuid):
def post(self, namespace_name, repository_name, uuid):
""" Queues a test notification for this repository. """
try:
test_note = model.notification.get_repo_notification(uuid)
except model.InvalidNotificationException:
raise NotFound()
test_note = model.queue_test_notification(uuid)
if (test_note.repository.namespace_user.username != namespace or
test_note.repository.name != repository):
raise NotFound()
if not test_note:
raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid))
event_info = NotificationEvent.get_event(test_note.event.name)
# TODO(jschorr): Stop depending on the worker module's data interface and instead only depend
# on the notification's data interface (to be added).
sample_data = event_info.get_sample_data(notification(test_note))
notification_data = build_notification_data(test_note, sample_data)
notification_queue.put([test_note.repository.namespace_user.username, repository,
test_note.event.name], json.dumps(notification_data))
return {}
return {}, 200

144
endpoints/api/repositorynotification_models_interface.py Normal file
View file

@ -0,0 +1,144 @@
import json
from abc import ABCMeta, abstractmethod
from collections import namedtuple
from six import add_metaclass
class RepositoryNotification(
namedtuple('RepositoryNotification', [
'uuid',
'title',
'event_name',
'method_name',
'config_json',
'event_config_json',
'number_of_failures',
])):
"""
RepositoryNotification represents a notification for a repository.
:type uuid: string
:type event: string
:type method: string
:type config: string
:type title: string
:type event_config: string
:type number_of_failures: int
"""
def to_dict(self):
try:
config = json.loads(self.config_json)
except ValueError:
config = {}
try:
event_config = json.loads(self.event_config_json)
except ValueError:
event_config = {}
return {
'uuid': self.uuid,
'title': self.title,
'event': self.event_name,
'method': self.method_name,
'config': config,
'event_config': event_config,
'number_of_failures': self.number_of_failures,
}
@add_metaclass(ABCMeta)
class RepoNotificationInterface(object):
"""
Interface that represents all data store interactions required by the RepositoryNotification API
"""
@abstractmethod
def create_repo_notification(self, namespace_name, repository_name, event_name, method_name, method_config, event_config, title=None):
"""
Args:
namespace_name: namespace of repository
repository_name: name of repository
event_name: name of event
method_name: name of method
method_config: method config, json string
event_config: event config, json string
title: title of the notification
Returns:
RepositoryNotification object
"""
pass
@abstractmethod
def list_repo_notifications(self, namespace_name, repository_name, event_name=None):
"""
Args:
namespace_name: namespace of repository
repository_name: name of repository
event_name: name of event
Returns:
list(RepositoryNotification)
"""
pass
@abstractmethod
def get_repo_notification(self, uuid):
"""
Args:
uuid: uuid of notification
Returns:
RepositoryNotification or None
"""
pass
@abstractmethod
def delete_repo_notification(self, namespace_name, repository_name, uuid):
"""
Args:
namespace_name: namespace of repository
repository_name: name of repository
uuid: uuid of notification
Returns:
RepositoryNotification or None
"""
pass
@abstractmethod
def reset_notification_number_of_failures(self, namespace_name, repository_name, uuid):
"""
Args:
namespace_name: namespace of repository
repository_name: name of repository
uuid: uuid of notification
Returns:
RepositoryNotification
"""
pass
@abstractmethod
def queue_test_notification(self, uuid):
"""
Args:
uuid: uuid of notification
Returns:
RepositoryNotification or None
"""
pass

71
endpoints/api/repositorynotification_models_pre_oci.py Normal file
View file

@ -0,0 +1,71 @@
import json
from app import notification_queue
from data import model
from data.model import InvalidNotificationException
from endpoints.api.repositorynotification_models_interface import RepoNotificationInterface, RepositoryNotification
from endpoints.notificationevent import NotificationEvent
from endpoints.notificationhelper import build_notification_data
class RepoNotificationPreOCIModel(RepoNotificationInterface):
def create_repo_notification(self, namespace_name, repository_name, event_name, method_name, method_config, event_config, title=None):
repository = model.repository.get_repository(namespace_name, repository_name)
return self._notification(model.notification.create_repo_notification(repository,
event_name,
method_name,
method_config,
event_config,
title))
def list_repo_notifications(self, namespace_name, repository_name, event_name=None):
return [self._notification(n)
for n in model.notification.list_repo_notifications(namespace_name, repository_name, event_name)]
def get_repo_notification(self, uuid):
try:
found = model.notification.get_repo_notification(uuid)
except InvalidNotificationException:
return None
return self._notification(found)
def delete_repo_notification(self, namespace_name, repository_name, uuid):
try:
found = model.notification.delete_repo_notification(namespace_name, repository_name, uuid)
except InvalidNotificationException:
return None
return self._notification(found)
def reset_notification_number_of_failures(self, namespace_name, repository_name, uuid):
return self._notification(
model.notification.reset_notification_number_of_failures(namespace_name, repository_name, uuid))
def queue_test_notification(self, uuid):
try:
notification = model.notification.get_repo_notification(uuid)
except InvalidNotificationException:
return None
event_info = NotificationEvent.get_event(notification.event.name)
sample_data = event_info.get_sample_data(notification)
notification_data = build_notification_data(notification, sample_data)
notification_queue.put([notification.repository.namespace_user.username, notification.uuid,
notification.event.name], json.dumps(notification_data))
return self._notification(notification)
def _notification(self, notification):
if not notification:
return None
return RepositoryNotification(uuid=notification.uuid,
title=notification.title,
event_name=notification.event.name,
method_name=notification.method.name,
config_json=notification.config_json,
event_config_json=notification.event_config_json,
number_of_failures=notification.number_of_failures)
pre_oci_model = RepoNotificationPreOCIModel()

84
endpoints/api/test/test_repositorynotification.py Normal file
View file

@ -0,0 +1,84 @@
import pytest
from mock import Mock, MagicMock
from endpoints.api.test.shared import conduct_api_call
from endpoints.api.repositorynotification import RepositoryNotificationList, RepositoryNotification, TestRepositoryNotification
from endpoints.test.shared import client_with_identity
import endpoints.api.repositorynotification_models_interface as iface
from test.fixtures import *
@pytest.fixture()
def authd_client(client):
with client_with_identity('devtable', client) as cl:
yield cl
def mock_get_notification(uuid):
mock_notification = MagicMock(iface.RepositoryNotification)
if uuid == 'exists':
mock_notification.return_value = iface.RepositoryNotification(
'exists',
'title',
'event_name',
'method_name',
'config_json',
'event_config_json',
2,
)
else:
mock_notification.return_value = None
return mock_notification
@pytest.mark.parametrize('namespace,repository,body,expected_code',[
('devtable', 'simple', dict(config={'url': 'http://example.com'}, event='repo_push',
method='webhook', eventConfig={}, title='test'), 201)
])
def test_create_repo_notification(namespace, repository, body, expected_code, authd_client):
params = {'repository': namespace + '/' + repository}
conduct_api_call(authd_client, RepositoryNotificationList, 'POST', params, body, expected_code=expected_code)
@pytest.mark.parametrize('namespace,repository,expected_code',[
('devtable', 'simple', 200)
])
def test_list_repo_notifications(namespace, repository, expected_code, authd_client):
params = {'repository': namespace + '/' + repository}
resp = conduct_api_call(authd_client, RepositoryNotificationList, 'GET', params, expected_code=expected_code).json
assert len(resp['notifications']) > 0
@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[
('devtable', 'simple', 'exists', 200),
('devtable', 'simple', 'not found', 404),
])
def test_get_repo_notification(namespace, repository, uuid, expected_code, authd_client, monkeypatch):
monkeypatch.setattr('endpoints.api.repositorynotification.model.get_repo_notification', mock_get_notification(uuid))
params = {'repository': namespace + '/' + repository, 'uuid': uuid}
conduct_api_call(authd_client, RepositoryNotification, 'GET', params, expected_code=expected_code)
@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[
('devtable', 'simple', 'exists', 204),
('devtable', 'simple', 'not found', 400),
])
def test_delete_repo_notification(namespace, repository, uuid, expected_code, authd_client, monkeypatch):
monkeypatch.setattr('endpoints.api.repositorynotification.model.delete_repo_notification', mock_get_notification(uuid))
params = {'repository': namespace + '/' + repository, 'uuid': uuid}
conduct_api_call(authd_client, RepositoryNotification, 'DELETE', params, expected_code=expected_code)
@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[
('devtable', 'simple', 'exists', 204),
('devtable', 'simple', 'not found', 400),
])
def test_reset_repo_noticiation(namespace, repository, uuid, expected_code, authd_client, monkeypatch):
monkeypatch.setattr('endpoints.api.repositorynotification.model.reset_notification_number_of_failures', mock_get_notification(uuid))
params = {'repository': namespace + '/' + repository, 'uuid': uuid}
conduct_api_call(authd_client, RepositoryNotification, 'POST', params, expected_code=expected_code)
@pytest.mark.parametrize('namespace,repository,uuid,expected_code',[
('devtable', 'simple', 'exists', 200),
('devtable', 'simple', 'not found', 400),
])
def test_test_repo_notification(namespace, repository, uuid, expected_code, authd_client, monkeypatch):
monkeypatch.setattr('endpoints.api.repositorynotification.model.queue_test_notification', mock_get_notification(uuid))
params = {'repository': namespace + '/' + repository, 'uuid': uuid}
conduct_api_call(authd_client, TestRepositoryNotification, 'POST', params, expected_code=expected_code)

2
endpoints/api/test/test_security.py
View file

@ -59,7 +59,7 @@ NOTIFICATION_PARAMS = {'namespace': 'devtable', 'repository': 'devtable/simple',
(RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, None, 403),
(RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'freshuser', 403),
(RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'reader', 403),
(RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'devtable', 204),
(RepositoryNotification, 'POST', NOTIFICATION_PARAMS, {}, 'devtable', 400),
(RepositoryTrust, 'POST', REPO_PARAMS, {'trust_enabled': True}, None, 403),
(RepositoryTrust, 'POST', REPO_PARAMS, {'trust_enabled': True}, 'freshuser', 403),

5
endpoints/notificationhelper.py
View file

@ -5,13 +5,14 @@ from contextlib import contextmanager
from app import app, notification_queue
from data import model
from auth.auth_context import get_authenticated_user, get_validated_oauth_token
from endpoints.notificationmethod import _get_namespace_name_from
DEFAULT_BATCH_SIZE = 1000
def build_event_data(repo, extra_data=None, subpage=None):
repo_string = '%s/%s' % (repo.namespace_name, repo.name)
repo_string = '%s/%s' % (_get_namespace_name_from(repo), repo.name)
homepage = '%s://%s/repository/%s' % (app.config['PREFERRED_URL_SCHEME'],
app.config['SERVER_HOSTNAME'],
repo_string)
@ -24,7 +25,7 @@ def build_event_data(repo, extra_data=None, subpage=None):
event_data = {
'repository': repo_string,
'namespace': repo.namespace_name,
'namespace': _get_namespace_name_from(repo),
'name': repo.name,
'docker_url': '%s/%s' % (app.config['SERVER_HOSTNAME'], repo_string),
'homepage': homepage,

21
endpoints/notificationmethod.py
View file

@ -56,7 +56,7 @@ class NotificationMethod(object):
"""
raise NotImplementedError
def validate(self, repository, config_data):
def validate(self, namespace_name, repository_name, config_data):
"""
Validates that the notification can be created with the given data. Throws
a CannotValidateNotificationMethodException on failure.
@ -88,12 +88,12 @@ class QuayNotificationMethod(NotificationMethod):
def method_name(cls):
return 'quay_notification'
def validate(self, repository, config_data):
status, err_message, target_users = self.find_targets(repository, config_data)
def validate(self, namespace_name, repository_name, config_data):
status, err_message, target_users = self.find_targets(namespace_name, repository_name, config_data)
if err_message:
raise CannotValidateNotificationMethodException(err_message)
def find_targets(self, repository, config_data):
def find_targets(self, namespace_name, repository_name, config_data):
target_info = config_data['target']
if target_info['kind'] == 'user':
@ -134,7 +134,7 @@ class QuayNotificationMethod(NotificationMethod):
# Lookup the target user or team to which we'll send the notification.
config_data = notification_obj.method_config_dict
status, err_message, target_users = self.find_targets(repository, config_data)
status, err_message, target_users = self.find_targets(_get_namespace_name_from(repository), repository.name, config_data)
if not status:
raise NotificationMethodPerformException(err_message)
@ -149,12 +149,11 @@ class EmailMethod(NotificationMethod):
def method_name(cls):
return 'email'
def validate(self, repository, config_data):
def validate(self, namespace_name, repository_name, config_data):
email = config_data.get('email', '')
if not email:
raise CannotValidateNotificationMethodException('Missing e-mail address')
record = model.repository.get_email_authorized_for_repo(_get_namespace_name_from(repository),
repository.name, email)
if not record or not record.confirmed:
@ -185,7 +184,7 @@ class WebhookMethod(NotificationMethod):
def method_name(cls):
return 'webhook'
def validate(self, repository, config_data):
def validate(self, namespace_name, repository_name, config_data):
url = config_data.get('url', '')
if not url:
raise CannotValidateNotificationMethodException('Missing webhook URL')
@ -222,7 +221,7 @@ class FlowdockMethod(NotificationMethod):
def method_name(cls):
return 'flowdock'
def validate(self, repository, config_data):
def validate(self, namespace_name, repository_name, config_data):
token = config_data.get('flow_api_token', '')
if not token:
raise CannotValidateNotificationMethodException('Missing Flowdock API Token')
@ -274,7 +273,7 @@ class HipchatMethod(NotificationMethod):
def method_name(cls):
return 'hipchat'
def validate(self, repository, config_data):
def validate(self, namespace_name, repository_name, config_data):
if not config_data.get('notification_token', ''):
raise CannotValidateNotificationMethodException('Missing Hipchat Room Notification Token')
@ -385,7 +384,7 @@ class SlackMethod(NotificationMethod):
def method_name(cls):
return 'slack'
def validate(self, repository, config_data):
def validate(self, namespace_name, repository_name, config_data):
if not config_data.get('url', ''):
raise CannotValidateNotificationMethodException('Missing Slack Callback URL')