diff --git a/buildman/jobutil/buildjob.py b/buildman/jobutil/buildjob.py index 02199898f..2f94b9cc6 100644 --- a/buildman/jobutil/buildjob.py +++ b/buildman/jobutil/buildjob.py @@ -2,7 +2,7 @@ import json import logging from cachetools import lru_cache -from endpoints.notificationhelper import spawn_notification +from notifications.notificationhelper import spawn_notification from data import model from util.imagetree import ImageTree from util.morecollections import AttrDict diff --git a/endpoints/api/repositorynotification.py b/endpoints/api/repositorynotification.py index 10192479b..34826fdb2 100644 --- a/endpoints/api/repositorynotification.py +++ b/endpoints/api/repositorynotification.py @@ -6,10 +6,11 @@ from flask import request from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin, log_action, validate_json_request, request_error, path_param, disallow_for_app_repositories) -from endpoints.exception import NotFound, InvalidRequest -from endpoints.notificationmethod import (NotificationMethod, - CannotValidateNotificationMethodException) -from endpoints.notificationhelper import build_notification_data +from endpoints.exception import NotFound +from notifications.notificationevent import NotificationEvent +from notifications.notificationmethod import (NotificationMethod, + CannotValidateNotificationMethodException) +from notifications.notificationhelper import build_notification_data from workers.notificationworker.models_pre_oci import notification from endpoints.api.repositorynotification_models_pre_oci import pre_oci_model as model @@ -61,18 +62,18 @@ class RepositoryNotificationList(RepositoryParamResource): @validate_json_request('NotificationCreateRequest') def post(self, namespace_name, repository_name): parsed = request.get_json() - + method_handler = NotificationMethod.get_method(parsed['method']) try: method_handler.validate(namespace_name, repository_name, parsed['config']) except CannotValidateNotificationMethodException as ex: raise request_error(message=ex.message) - - new_notification = model.create_repo_notification(namespace_name, repository_name, - parsed['event'], - parsed['method'], - parsed['config'], - parsed['eventConfig'], + + new_notification = model.create_repo_notification(namespace_name, repository_name, + parsed['event'], + parsed['method'], + parsed['config'], + parsed['eventConfig'], parsed.get('title')) log_action('add_repo_notification', namespace_name, @@ -116,7 +117,7 @@ class RepositoryNotification(RepositoryParamResource): deleted = model.delete_repo_notification(namespace_name, repository_name, uuid) if not deleted: raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid)) - + log_action('delete_repo_notification', namespace_name, {'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid, 'event': deleted.event_name, 'method': deleted.method_name}, @@ -132,7 +133,7 @@ class RepositoryNotification(RepositoryParamResource): reset = model.reset_notification_number_of_failures(namespace_name, repository_name, uuid) if not reset: raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid)) - + log_action('reset_repo_notification', namespace_name, {'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid, 'event': reset.event_name, 'method': reset.method_name}, @@ -155,5 +156,5 @@ class TestRepositoryNotification(RepositoryParamResource): if not test_note: raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid)) - + return {}, 200 diff --git a/endpoints/building.py b/endpoints/building.py index 74a611841..470a1ab84 100644 --- a/endpoints/building.py +++ b/endpoints/building.py @@ -9,7 +9,7 @@ from app import app, dockerfile_build_queue, metric_queue from data import model from data.database import db from auth.auth_context import get_authenticated_user -from endpoints.notificationhelper import spawn_notification +from notifications.notificationhelper import spawn_notification from util.names import escape_tag from util.morecollections import AttrDict diff --git a/endpoints/v1/index.py b/endpoints/v1/index.py index 877fb90c5..25678a924 100644 --- a/endpoints/v1/index.py +++ b/endpoints/v1/index.py @@ -17,6 +17,7 @@ from endpoints.decorators import anon_protect, anon_allowed, parse_repository_na from endpoints.notificationhelper import spawn_notification from endpoints.v1 import v1_bp from endpoints.v1.models_pre_oci import pre_oci_model as model +from notifications.notificationhelper import spawn_notification from util.audit import track_and_log from util.http import abort from util.names import REPOSITORY_NAME_REGEX diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index b4970e2f6..ae04524dd 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -20,6 +20,7 @@ from endpoints.v2.labelhandlers import handle_label from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES +from notifications.notificationhelper import spawn_notification from util.audit import track_and_log from util.names import VALID_TAG_PATTERN from util.registry.replication import queue_replication_batch diff --git a/notifications/__init__.py b/notifications/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/notifications/models_interface.py b/notifications/models_interface.py new file mode 100644 index 000000000..4c5dac3bd --- /dev/null +++ b/notifications/models_interface.py @@ -0,0 +1,15 @@ +from collections import namedtuple + +class Repository(namedtuple('Repository', ['namespace_name', 'name'])): + """ + Repository represents a repository. + """ + + +class Notification( + namedtuple('Notification', [ + 'uuid', 'event_name', 'method_name', 'event_config_dict', 'method_config_dict', + 'repository'])): + """ + Notification represents a registered notification of some kind. + """ diff --git a/endpoints/notificationevent.py b/notifications/notificationevent.py similarity index 98% rename from endpoints/notificationevent.py rename to notifications/notificationevent.py index 750e84caf..0401e1b05 100644 --- a/endpoints/notificationevent.py +++ b/notifications/notificationevent.py @@ -1,17 +1,16 @@ import logging import time -import json import re from datetime import datetime -from endpoints.notificationhelper import build_event_data +from notifications.notificationhelper import build_event_data from util.jinjautil import get_template_env -from util.morecollections import AttrDict from util.secscan import PRIORITY_LEVELS, get_priority_for_index -template_env = get_template_env("events") logger = logging.getLogger(__name__) +TEMPLATE_ENV = get_template_env("events") + class InvalidNotificationEventException(Exception): pass @@ -36,7 +35,7 @@ class NotificationEvent(object): """ Returns a human readable HTML message for the given notification data. """ - return template_env.get_template(self.event_name() + '.html').render({ + return TEMPLATE_ENV.get_template(self.event_name() + '.html').render({ 'event_data': event_data, 'notification_data': notification_data }) @@ -363,4 +362,3 @@ class BuildCancelledEvent(BaseBuildEvent): def get_summary(self, event_data, notification_data): return 'Build cancelled ' + _build_summary(event_data) - diff --git a/endpoints/notificationhelper.py b/notifications/notificationhelper.py similarity index 100% rename from endpoints/notificationhelper.py rename to notifications/notificationhelper.py diff --git a/endpoints/notificationmethod.py b/notifications/notificationmethod.py similarity index 83% rename from endpoints/notificationmethod.py rename to notifications/notificationmethod.py index 19c6ad9a5..2a426cfa6 100644 --- a/endpoints/notificationmethod.py +++ b/notifications/notificationmethod.py @@ -1,6 +1,6 @@ -import json import logging import re +import json import requests from flask_mail import Message @@ -27,22 +27,11 @@ class NotificationMethodPerformException(JobException): pass -def _get_namespace_name_from(repository): - # TODO Charlie 2017-07-14: This is hack for a bug in production - # because in some places have started calling this method with - # pre oci models and in some we have started calling with non pre oci models. We should - # remove this when we have switched over to database interfaces. - if hasattr(repository, 'namespace_name'): - namespace_name = repository.namespace_name - else: - namespace_name = repository.namespace_user.username - return namespace_name +def _ssl_cert(): + if app.config['PREFERRED_URL_SCHEME'] == 'https': + return [OVERRIDE_CONFIG_DIRECTORY + f for f in SSL_FILENAMES] - -SSLClientCert = None -if app.config['PREFERRED_URL_SCHEME'] == 'https': - # TODO(jschorr): move this into the config provider library - SSLClientCert = [OVERRIDE_CONFIG_DIRECTORY + f for f in SSL_FILENAMES] + return None class NotificationMethod(object): @@ -56,7 +45,7 @@ class NotificationMethod(object): """ raise NotImplementedError - def validate(self, namespace_name, repository_name, config_data): + def validate(self, repository, config_data): """ Validates that the notification can be created with the given data. Throws a CannotValidateNotificationMethodException on failure. @@ -88,12 +77,12 @@ class QuayNotificationMethod(NotificationMethod): def method_name(cls): return 'quay_notification' - def validate(self, namespace_name, repository_name, config_data): - status, err_message, target_users = self.find_targets(namespace_name, repository_name, config_data) + def validate(self, repository, config_data): + _, err_message, _ = self.find_targets(repository, config_data) if err_message: raise CannotValidateNotificationMethodException(err_message) - def find_targets(self, namespace_name, repository_name, config_data): + def find_targets(self, repository, config_data): target_info = config_data['target'] if target_info['kind'] == 'user': @@ -110,7 +99,7 @@ class QuayNotificationMethod(NotificationMethod): return (True, 'Unknown organization %s' % target_info['name'], None) # Only repositories under the organization can cause notifications to that org. - if target_info['name'] != _get_namespace_name_from(repository): + if target_info['name'] != repository.namespace_name: return (False, 'Organization name must match repository namespace') return (True, None, [target]) @@ -118,7 +107,7 @@ class QuayNotificationMethod(NotificationMethod): # Lookup the team. org_team = None try: - org_team = model.team.get_organization_team(_get_namespace_name_from(repository), target_info['name']) + org_team = model.team.get_organization_team(repository.namespace_name, target_info['name']) except model.InvalidTeamException: # Probably deleted. return (True, 'Unknown team %s' % target_info['name'], None) @@ -134,7 +123,7 @@ class QuayNotificationMethod(NotificationMethod): # Lookup the target user or team to which we'll send the notification. config_data = notification_obj.method_config_dict - status, err_message, target_users = self.find_targets(_get_namespace_name_from(repository), repository.name, config_data) + status, err_message, target_users = self.find_targets(repository, config_data) if not status: raise NotificationMethodPerformException(err_message) @@ -149,12 +138,12 @@ class EmailMethod(NotificationMethod): def method_name(cls): return 'email' - def validate(self, namespace_name, repository_name, config_data): + def validate(self, repository, config_data): email = config_data.get('email', '') if not email: raise CannotValidateNotificationMethodException('Missing e-mail address') - record = model.repository.get_email_authorized_for_repo(_get_namespace_name_from(repository), + record = model.repository.get_email_authorized_for_repo(repository.namespace_name, repository.name, email) if not record or not record.confirmed: raise CannotValidateNotificationMethodException('The specified e-mail address ' @@ -175,7 +164,7 @@ class EmailMethod(NotificationMethod): try: mail.send(msg) except Exception as ex: - logger.exception('Email was unable to be sent: %s' % ex.message) + logger.exception('Email was unable to be sent') raise NotificationMethodPerformException(ex.message) @@ -184,7 +173,7 @@ class WebhookMethod(NotificationMethod): def method_name(cls): return 'webhook' - def validate(self, namespace_name, repository_name, config_data): + def validate(self, repository, config_data): url = config_data.get('url', '') if not url: raise CannotValidateNotificationMethodException('Missing webhook URL') @@ -199,7 +188,7 @@ class WebhookMethod(NotificationMethod): headers = {'Content-type': 'application/json'} try: - resp = requests.post(url, data=json.dumps(payload), headers=headers, cert=SSLClientCert, + resp = requests.post(url, data=json.dumps(payload), headers=headers, cert=_ssl_cert(), timeout=METHOD_TIMEOUT) if resp.status_code / 100 != 2: error_message = '%s response for webhook to url: %s' % (resp.status_code, url) @@ -208,7 +197,7 @@ class WebhookMethod(NotificationMethod): raise NotificationMethodPerformException(error_message) except requests.exceptions.RequestException as ex: - logger.exception('Webhook was unable to be sent: %s' % ex.message) + logger.exception('Webhook was unable to be sent') raise NotificationMethodPerformException(ex.message) @@ -221,7 +210,7 @@ class FlowdockMethod(NotificationMethod): def method_name(cls): return 'flowdock' - def validate(self, namespace_name, repository_name, config_data): + def validate(self, repository, config_data): token = config_data.get('flow_api_token', '') if not token: raise CannotValidateNotificationMethodException('Missing Flowdock API Token') @@ -232,7 +221,7 @@ class FlowdockMethod(NotificationMethod): if not token: return - owner = model.user.get_user_or_org(_get_namespace_name_from(notification_obj.repository)) + owner = model.user.get_user_or_org(notification_obj.repository.namespace_name) if not owner: # Something went wrong. return @@ -245,7 +234,7 @@ class FlowdockMethod(NotificationMethod): 'subject': event_handler.get_summary(notification_data['event_data'], notification_data), 'content': event_handler.get_message(notification_data['event_data'], notification_data), 'from_name': owner.username, - 'project': (_get_namespace_name_from(notification_obj.repository)+ ' ' + + 'project': (notification_obj.repository.namespace_name + ' ' + notification_obj.repository.name), 'tags': ['#' + event_handler.event_name()], 'link': notification_data['event_data']['homepage'] @@ -260,7 +249,7 @@ class FlowdockMethod(NotificationMethod): raise NotificationMethodPerformException(error_message) except requests.exceptions.RequestException as ex: - logger.exception('Flowdock method was unable to be sent: %s' % ex.message) + logger.exception('Flowdock method was unable to be sent') raise NotificationMethodPerformException(ex.message) @@ -273,7 +262,7 @@ class HipchatMethod(NotificationMethod): def method_name(cls): return 'hipchat' - def validate(self, namespace_name, repository_name, config_data): + def validate(self, repository, config_data): if not config_data.get('notification_token', ''): raise CannotValidateNotificationMethodException('Missing Hipchat Room Notification Token') @@ -288,7 +277,7 @@ class HipchatMethod(NotificationMethod): if not token or not room_id: return - owner = model.user.get_user_or_org(_get_namespace_name_from(notification_obj.repository)) + owner = model.user.get_user_or_org(notification_obj.repository.namespace_name) if not owner: # Something went wrong. return @@ -321,7 +310,7 @@ class HipchatMethod(NotificationMethod): raise NotificationMethodPerformException(error_message) except requests.exceptions.RequestException as ex: - logger.exception('Hipchat method was unable to be sent: %s' % ex.message) + logger.exception('Hipchat method was unable to be sent') raise NotificationMethodPerformException(ex.message) @@ -384,7 +373,7 @@ class SlackMethod(NotificationMethod): def method_name(cls): return 'slack' - def validate(self, namespace_name, repository_name, config_data): + def validate(self, repository, config_data): if not config_data.get('url', ''): raise CannotValidateNotificationMethodException('Missing Slack Callback URL') @@ -400,7 +389,7 @@ class SlackMethod(NotificationMethod): if not url: return - owner = model.user.get_user_or_org(_get_namespace_name_from(notification_obj.repository)) + owner = model.user.get_user_or_org(notification_obj.repository.namespace_name) if not owner: # Something went wrong. return diff --git a/endpoints/test/test_notificationevent.py b/notifications/test/test_notificationevent.py similarity index 92% rename from endpoints/test/test_notificationevent.py rename to notifications/test/test_notificationevent.py index fd4d81c12..7429ef0a6 100644 --- a/endpoints/test/test_notificationevent.py +++ b/notifications/test/test_notificationevent.py @@ -1,6 +1,4 @@ -import json - -from endpoints.notificationevent import NotificationEvent +from notifications.notificationevent import NotificationEvent from util.morecollections import AttrDict from test.fixtures import * diff --git a/test/test_notifications.py b/test/test_notifications.py index 25ad26ae7..486bef19d 100644 --- a/test/test_notifications.py +++ b/test/test_notifications.py @@ -1,7 +1,7 @@ import unittest -from endpoints.notificationevent import (BuildSuccessEvent, NotificationEvent, - VulnerabilityFoundEvent) +from notifications.notificationevent import (BuildSuccessEvent, NotificationEvent, + VulnerabilityFoundEvent) from util.morecollections import AttrDict class TestCreate(unittest.TestCase): diff --git a/test/test_secscan.py b/test/test_secscan.py index 1e312e00d..8ba559d95 100644 --- a/test/test_secscan.py +++ b/test/test_secscan.py @@ -5,9 +5,9 @@ import unittest from app import app, storage, notification_queue from data import model from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION -from endpoints.notificationevent import VulnerabilityFoundEvent from endpoints.v2 import v2_bp from initdb import setup_database_for_testing, finished_database_for_testing +from notifications.notificationevent import VulnerabilityFoundEvent from util.morecollections import AttrDict from util.secscan.api import SecurityScannerAPI, APIRequestFailure from util.secscan.analyzer import LayerAnalyzer diff --git a/util/secscan/analyzer.py b/util/secscan/analyzer.py index cdfe99051..0b4698c46 100644 --- a/util/secscan/analyzer.py +++ b/util/secscan/analyzer.py @@ -5,10 +5,10 @@ from collections import defaultdict import features -from endpoints.notificationhelper import spawn_notification from data.database import ExternalNotificationEvent, IMAGE_NOT_SCANNED_ENGINE_VERSION, Image from data.model.tag import filter_tags_have_repository_event, get_tags_for_image from data.model.image import set_secscan_status, get_image_with_storage_and_parent_base +from notifications.notificationhelper import spawn_notification from util.secscan import PRIORITY_LEVELS from util.secscan.api import (APIRequestFailure, AnalyzeLayerException, MissingParentLayerException, InvalidLayerException, AnalyzeLayerRetryException) diff --git a/util/secscan/notifier.py b/util/secscan/notifier.py index c71209ebe..bbd6525be 100644 --- a/util/secscan/notifier.py +++ b/util/secscan/notifier.py @@ -10,7 +10,7 @@ from data.model.tag import (filter_has_repository_event, filter_tags_have_reposi from data.database import (Image, ImageStorage, ExternalNotificationEvent, Repository, RepositoryTag) -from endpoints.notificationhelper import notification_batch +from notifications.notificationhelper import notification_batch from util.secscan import PRIORITY_LEVELS from util.secscan.api import APIRequestFailure from util.morecollections import AttrDict, StreamingDiffTracker, IndexedStreamingDiffTracker diff --git a/workers/notificationworker/notificationworker.py b/workers/notificationworker/notificationworker.py index 9a478f758..c4c0ba907 100644 --- a/workers/notificationworker/notificationworker.py +++ b/workers/notificationworker/notificationworker.py @@ -1,8 +1,8 @@ import logging from app import notification_queue -from endpoints.notificationmethod import NotificationMethod, InvalidNotificationMethodException -from endpoints.notificationevent import NotificationEvent, InvalidNotificationEventException +from notifications.notificationmethod import NotificationMethod, InvalidNotificationMethodException +from notifications.notificationevent import NotificationEvent, InvalidNotificationEventException from workers.notificationworker.models_pre_oci import pre_oci_model as model from workers.queueworker import QueueWorker, JobException