Move notifications into its own package

buildman/jobutil/buildjob.py

@@ -2,7 +2,7 @@ import json
 import logging
 
 from cachetools import lru_cache
-from endpoints.notificationhelper import spawn_notification
+from notifications.notificationhelper import spawn_notification
 from data import model
 from util.imagetree import ImageTree
 from util.morecollections import AttrDict

endpoints/api/repositorynotification.py

@@ -6,10 +6,11 @@ from flask import request
 from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin,
                            log_action, validate_json_request, request_error,
                            path_param, disallow_for_app_repositories)
-from endpoints.exception import NotFound, InvalidRequest
-from endpoints.notificationmethod import (NotificationMethod,
-                                          CannotValidateNotificationMethodException)
-from endpoints.notificationhelper import build_notification_data
+from endpoints.exception import NotFound
+from notifications.notificationevent import NotificationEvent
+from notifications.notificationmethod import (NotificationMethod,
+                                              CannotValidateNotificationMethodException)
+from notifications.notificationhelper import build_notification_data
 from workers.notificationworker.models_pre_oci import notification
 from endpoints.api.repositorynotification_models_pre_oci import pre_oci_model as model
 
@@ -61,18 +62,18 @@ class RepositoryNotificationList(RepositoryParamResource):
   @validate_json_request('NotificationCreateRequest')
   def post(self, namespace_name, repository_name):
     parsed = request.get_json()
-    
+
     method_handler = NotificationMethod.get_method(parsed['method'])
     try:
       method_handler.validate(namespace_name, repository_name, parsed['config'])
     except CannotValidateNotificationMethodException as ex:
       raise request_error(message=ex.message)
-    
-    new_notification = model.create_repo_notification(namespace_name, repository_name, 
-                                                      parsed['event'], 
-                                                      parsed['method'], 
-                                                      parsed['config'], 
-                                                      parsed['eventConfig'], 
+
+    new_notification = model.create_repo_notification(namespace_name, repository_name,
+                                                      parsed['event'],
+                                                      parsed['method'],
+                                                      parsed['config'],
+                                                      parsed['eventConfig'],
                                                       parsed.get('title'))
 
     log_action('add_repo_notification', namespace_name,
@@ -116,7 +117,7 @@ class RepositoryNotification(RepositoryParamResource):
     deleted = model.delete_repo_notification(namespace_name, repository_name, uuid)
     if not deleted:
       raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid))
-    
+
     log_action('delete_repo_notification', namespace_name,
                {'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid,
                 'event': deleted.event_name, 'method': deleted.method_name},
@@ -132,7 +133,7 @@ class RepositoryNotification(RepositoryParamResource):
     reset = model.reset_notification_number_of_failures(namespace_name, repository_name, uuid)
     if not reset:
       raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid))
-    
+
     log_action('reset_repo_notification', namespace_name,
                {'repo': repository_name, 'namespace': namespace_name, 'notification_id': uuid,
                 'event': reset.event_name, 'method': reset.method_name},
@@ -155,5 +156,5 @@ class TestRepositoryNotification(RepositoryParamResource):
 
     if not test_note:
       raise InvalidRequest("No repository notification found for: %s, %s, %s" % (namespace_name, repository_name, uuid))
-    
+
     return {}, 200

endpoints/building.py

@@ -9,7 +9,7 @@ from app import app, dockerfile_build_queue, metric_queue
 from data import model
 from data.database import db
 from auth.auth_context import get_authenticated_user
-from endpoints.notificationhelper import spawn_notification
+from notifications.notificationhelper import spawn_notification
 from util.names import escape_tag
 from util.morecollections import AttrDict
 

endpoints/v1/index.py

@@ -17,6 +17,7 @@ from endpoints.decorators import anon_protect, anon_allowed, parse_repository_na
 from endpoints.notificationhelper import spawn_notification
 from endpoints.v1 import v1_bp
 from endpoints.v1.models_pre_oci import pre_oci_model as model
+from notifications.notificationhelper import spawn_notification
 from util.audit import track_and_log
 from util.http import abort
 from util.names import REPOSITORY_NAME_REGEX

endpoints/v2/manifest.py

@@ -20,6 +20,7 @@ from endpoints.v2.labelhandlers import handle_label
 from image.docker import ManifestException
 from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
 from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES
+from notifications.notificationhelper import spawn_notification
 from util.audit import track_and_log
 from util.names import VALID_TAG_PATTERN
 from util.registry.replication import queue_replication_batch

notifications/models_interface.py

@@ -0,0 +1,15 @@
+from collections import namedtuple
+
+class Repository(namedtuple('Repository', ['namespace_name', 'name'])):
+  """
+  Repository represents a repository.
+  """
+
+
+class Notification(
+    namedtuple('Notification', [
+      'uuid', 'event_name', 'method_name', 'event_config_dict', 'method_config_dict',
+      'repository'])):
+  """
+  Notification represents a registered notification of some kind.
+  """

notifications/notificationevent.py

@@ -1,17 +1,16 @@
 import logging
 import time
-import json
 import re
 
 from datetime import datetime
-from endpoints.notificationhelper import build_event_data
+from notifications.notificationhelper import build_event_data
 from util.jinjautil import get_template_env
-from util.morecollections import AttrDict
 from util.secscan import PRIORITY_LEVELS, get_priority_for_index
 
-template_env = get_template_env("events")
 logger = logging.getLogger(__name__)
 
+TEMPLATE_ENV = get_template_env("events")
+
 class InvalidNotificationEventException(Exception):
   pass
 
@@ -36,7 +35,7 @@ class NotificationEvent(object):
     """
     Returns a human readable HTML message for the given notification data.
     """
-    return template_env.get_template(self.event_name() + '.html').render({
+    return TEMPLATE_ENV.get_template(self.event_name() + '.html').render({
       'event_data': event_data,
       'notification_data': notification_data
     })
@@ -363,4 +362,3 @@ class BuildCancelledEvent(BaseBuildEvent):
 
   def get_summary(self, event_data, notification_data):
     return 'Build cancelled ' + _build_summary(event_data)
-

notifications/notificationmethod.py

@@ -1,6 +1,6 @@
-import json
 import logging
 import re
+import json
 
 import requests
 from flask_mail import Message
@@ -27,22 +27,11 @@ class NotificationMethodPerformException(JobException):
   pass
 
 
-def _get_namespace_name_from(repository):
-  # TODO Charlie 2017-07-14: This is hack for a bug in production
-  # because in some places have started calling this method with
-  # pre oci models and in some we have started calling with non pre oci models. We should
-  # remove this when we have switched over to database interfaces.
-  if hasattr(repository, 'namespace_name'):
-    namespace_name = repository.namespace_name
-  else:
-    namespace_name = repository.namespace_user.username
-  return namespace_name
+def _ssl_cert():
+  if app.config['PREFERRED_URL_SCHEME'] == 'https':
+    return [OVERRIDE_CONFIG_DIRECTORY + f for f in SSL_FILENAMES]
 
-
-SSLClientCert = None
-if app.config['PREFERRED_URL_SCHEME'] == 'https':
-  # TODO(jschorr): move this into the config provider library
-  SSLClientCert = [OVERRIDE_CONFIG_DIRECTORY + f for f in SSL_FILENAMES]
+  return None
 
 
 class NotificationMethod(object):
@@ -56,7 +45,7 @@ class NotificationMethod(object):
     """
     raise NotImplementedError
 
-  def validate(self, namespace_name, repository_name, config_data):
+  def validate(self, repository, config_data):
     """
     Validates that the notification can be created with the given data. Throws
     a CannotValidateNotificationMethodException on failure.
@@ -88,12 +77,12 @@ class QuayNotificationMethod(NotificationMethod):
   def method_name(cls):
     return 'quay_notification'
 
-  def validate(self, namespace_name, repository_name, config_data):
-    status, err_message, target_users = self.find_targets(namespace_name, repository_name, config_data)
+  def validate(self, repository, config_data):
+    _, err_message, _ = self.find_targets(repository, config_data)
     if err_message:
       raise CannotValidateNotificationMethodException(err_message)
 
-  def find_targets(self, namespace_name, repository_name, config_data):
+  def find_targets(self, repository, config_data):
     target_info = config_data['target']
 
     if target_info['kind'] == 'user':
@@ -110,7 +99,7 @@ class QuayNotificationMethod(NotificationMethod):
         return (True, 'Unknown organization %s' % target_info['name'], None)
 
       # Only repositories under the organization can cause notifications to that org.
-      if target_info['name'] != _get_namespace_name_from(repository):
+      if target_info['name'] != repository.namespace_name:
         return (False, 'Organization name must match repository namespace')
 
       return (True, None, [target])
@@ -118,7 +107,7 @@ class QuayNotificationMethod(NotificationMethod):
       # Lookup the team.
       org_team = None
       try:
-        org_team = model.team.get_organization_team(_get_namespace_name_from(repository), target_info['name'])
+        org_team = model.team.get_organization_team(repository.namespace_name, target_info['name'])
       except model.InvalidTeamException:
         # Probably deleted.
         return (True, 'Unknown team %s' % target_info['name'], None)
@@ -134,7 +123,7 @@ class QuayNotificationMethod(NotificationMethod):
 
     # Lookup the target user or team to which we'll send the notification.
     config_data = notification_obj.method_config_dict
-    status, err_message, target_users = self.find_targets(_get_namespace_name_from(repository), repository.name, config_data)
+    status, err_message, target_users = self.find_targets(repository, config_data)
     if not status:
       raise NotificationMethodPerformException(err_message)
 
@@ -149,12 +138,12 @@ class EmailMethod(NotificationMethod):
   def method_name(cls):
     return 'email'
 
-  def validate(self, namespace_name, repository_name, config_data):
+  def validate(self, repository, config_data):
     email = config_data.get('email', '')
     if not email:
       raise CannotValidateNotificationMethodException('Missing e-mail address')
 
-    record = model.repository.get_email_authorized_for_repo(_get_namespace_name_from(repository),
+    record = model.repository.get_email_authorized_for_repo(repository.namespace_name,
                                                             repository.name, email)
     if not record or not record.confirmed:
       raise CannotValidateNotificationMethodException('The specified e-mail address '
@@ -175,7 +164,7 @@ class EmailMethod(NotificationMethod):
       try:
         mail.send(msg)
       except Exception as ex:
-        logger.exception('Email was unable to be sent: %s' % ex.message)
+        logger.exception('Email was unable to be sent')
         raise NotificationMethodPerformException(ex.message)
 
 
@@ -184,7 +173,7 @@ class WebhookMethod(NotificationMethod):
   def method_name(cls):
     return 'webhook'
 
-  def validate(self, namespace_name, repository_name, config_data):
+  def validate(self, repository, config_data):
     url = config_data.get('url', '')
     if not url:
       raise CannotValidateNotificationMethodException('Missing webhook URL')
@@ -199,7 +188,7 @@ class WebhookMethod(NotificationMethod):
     headers = {'Content-type': 'application/json'}
 
     try:
-      resp = requests.post(url, data=json.dumps(payload), headers=headers, cert=SSLClientCert,
+      resp = requests.post(url, data=json.dumps(payload), headers=headers, cert=_ssl_cert(),
                            timeout=METHOD_TIMEOUT)
       if resp.status_code / 100 != 2:
         error_message = '%s response for webhook to url: %s' % (resp.status_code, url)
@@ -208,7 +197,7 @@ class WebhookMethod(NotificationMethod):
         raise NotificationMethodPerformException(error_message)
 
     except requests.exceptions.RequestException as ex:
-      logger.exception('Webhook was unable to be sent: %s' % ex.message)
+      logger.exception('Webhook was unable to be sent')
       raise NotificationMethodPerformException(ex.message)
 
 
@@ -221,7 +210,7 @@ class FlowdockMethod(NotificationMethod):
   def method_name(cls):
     return 'flowdock'
 
-  def validate(self, namespace_name, repository_name, config_data):
+  def validate(self, repository, config_data):
     token = config_data.get('flow_api_token', '')
     if not token:
       raise CannotValidateNotificationMethodException('Missing Flowdock API Token')
@@ -232,7 +221,7 @@ class FlowdockMethod(NotificationMethod):
     if not token:
       return
 
-    owner = model.user.get_user_or_org(_get_namespace_name_from(notification_obj.repository))
+    owner = model.user.get_user_or_org(notification_obj.repository.namespace_name)
     if not owner:
       # Something went wrong.
       return
@@ -245,7 +234,7 @@ class FlowdockMethod(NotificationMethod):
       'subject': event_handler.get_summary(notification_data['event_data'], notification_data),
       'content': event_handler.get_message(notification_data['event_data'], notification_data),
       'from_name': owner.username,
-      'project': (_get_namespace_name_from(notification_obj.repository)+ ' ' +
+      'project': (notification_obj.repository.namespace_name + ' ' +
                   notification_obj.repository.name),
       'tags': ['#' + event_handler.event_name()],
       'link': notification_data['event_data']['homepage']
@@ -260,7 +249,7 @@ class FlowdockMethod(NotificationMethod):
         raise NotificationMethodPerformException(error_message)
 
     except requests.exceptions.RequestException as ex:
-      logger.exception('Flowdock method was unable to be sent: %s' % ex.message)
+      logger.exception('Flowdock method was unable to be sent')
       raise NotificationMethodPerformException(ex.message)
 
 
@@ -273,7 +262,7 @@ class HipchatMethod(NotificationMethod):
   def method_name(cls):
     return 'hipchat'
 
-  def validate(self, namespace_name, repository_name, config_data):
+  def validate(self, repository, config_data):
     if not config_data.get('notification_token', ''):
       raise CannotValidateNotificationMethodException('Missing Hipchat Room Notification Token')
 
@@ -288,7 +277,7 @@ class HipchatMethod(NotificationMethod):
     if not token or not room_id:
       return
 
-    owner = model.user.get_user_or_org(_get_namespace_name_from(notification_obj.repository))
+    owner = model.user.get_user_or_org(notification_obj.repository.namespace_name)
     if not owner:
       # Something went wrong.
       return
@@ -321,7 +310,7 @@ class HipchatMethod(NotificationMethod):
         raise NotificationMethodPerformException(error_message)
 
     except requests.exceptions.RequestException as ex:
-      logger.exception('Hipchat method was unable to be sent: %s' % ex.message)
+      logger.exception('Hipchat method was unable to be sent')
       raise NotificationMethodPerformException(ex.message)
 
 
@@ -384,7 +373,7 @@ class SlackMethod(NotificationMethod):
   def method_name(cls):
     return 'slack'
 
-  def validate(self, namespace_name, repository_name, config_data):
+  def validate(self, repository, config_data):
     if not config_data.get('url', ''):
       raise CannotValidateNotificationMethodException('Missing Slack Callback URL')
 
@@ -400,7 +389,7 @@ class SlackMethod(NotificationMethod):
     if not url:
       return
 
-    owner = model.user.get_user_or_org(_get_namespace_name_from(notification_obj.repository))
+    owner = model.user.get_user_or_org(notification_obj.repository.namespace_name)
     if not owner:
       # Something went wrong.
       return

notifications/test/test_notificationevent.py

@@ -1,6 +1,4 @@
-import json
-
-from endpoints.notificationevent import NotificationEvent
+from notifications.notificationevent import NotificationEvent
 from util.morecollections import AttrDict
 
 from test.fixtures import *

test/test_notifications.py

@@ -1,7 +1,7 @@
 import unittest
 
-from endpoints.notificationevent import (BuildSuccessEvent, NotificationEvent,
-                                         VulnerabilityFoundEvent)
+from notifications.notificationevent import (BuildSuccessEvent, NotificationEvent,
+                                             VulnerabilityFoundEvent)
 from util.morecollections import AttrDict
 
 class TestCreate(unittest.TestCase):

test/test_secscan.py

@@ -5,9 +5,9 @@ import unittest
 from app import app, storage, notification_queue
 from data import model
 from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
-from endpoints.notificationevent import VulnerabilityFoundEvent
 from endpoints.v2 import v2_bp
 from initdb import setup_database_for_testing, finished_database_for_testing
+from notifications.notificationevent import VulnerabilityFoundEvent
 from util.morecollections import AttrDict
 from util.secscan.api import SecurityScannerAPI, APIRequestFailure
 from util.secscan.analyzer import LayerAnalyzer

util/secscan/analyzer.py

@@ -5,10 +5,10 @@ from collections import defaultdict
 
 import features
 
-from endpoints.notificationhelper import spawn_notification
 from data.database import ExternalNotificationEvent, IMAGE_NOT_SCANNED_ENGINE_VERSION, Image
 from data.model.tag import filter_tags_have_repository_event, get_tags_for_image
 from data.model.image import set_secscan_status, get_image_with_storage_and_parent_base
+from notifications.notificationhelper import spawn_notification
 from util.secscan import PRIORITY_LEVELS
 from util.secscan.api import (APIRequestFailure, AnalyzeLayerException, MissingParentLayerException,
                               InvalidLayerException, AnalyzeLayerRetryException)

util/secscan/notifier.py

@@ -10,7 +10,7 @@ from data.model.tag import (filter_has_repository_event, filter_tags_have_reposi
 
 from data.database import (Image, ImageStorage, ExternalNotificationEvent, Repository,
                            RepositoryTag)
-from endpoints.notificationhelper import notification_batch
+from notifications.notificationhelper import notification_batch
 from util.secscan import PRIORITY_LEVELS
 from util.secscan.api import APIRequestFailure
 from util.morecollections import AttrDict, StreamingDiffTracker, IndexedStreamingDiffTracker

workers/notificationworker/notificationworker.py

@@ -1,8 +1,8 @@
 import logging
 
 from app import notification_queue
-from endpoints.notificationmethod import NotificationMethod, InvalidNotificationMethodException
-from endpoints.notificationevent import NotificationEvent, InvalidNotificationEventException
+from notifications.notificationmethod import NotificationMethod, InvalidNotificationMethodException
+from notifications.notificationevent import NotificationEvent, InvalidNotificationEventException
 from workers.notificationworker.models_pre_oci import pre_oci_model as model
 from workers.queueworker import QueueWorker, JobException