initial import for Open Source 🎉

util/secscan/__init__.py

@@ -0,0 +1,109 @@
+# NOTE: This objects are used directly in the external-notification-data and vulnerability-service
+# on the frontend, so be careful with changing their existing keys.
+PRIORITY_LEVELS = {
+  'Unknown': {
+    'title': 'Unknown',
+    'index': 6,
+    'level': 'info',
+    'color': '#9B9B9B',
+    'score': 0,
+
+    'description': 'Unknown is either a security problem that has not been assigned to a priority' +
+                   ' yet or a priority that our system did not recognize',
+    'banner_required': False
+  },
+
+  'Negligible': {
+    'title': 'Negligible',
+    'index': 5,
+    'level': 'info',
+    'color': '#9B9B9B',
+    'score': 1,
+
+    'description': 'Negligible is technically a security problem, but is only theoretical ' +
+                   'in nature, requires a very special situation, has almost no install base, ' +
+                   'or does no real damage.',
+    'banner_required': False
+  },
+
+  'Low': {
+    'title': 'Low',
+    'index': 4,
+    'level': 'warning',
+    'color': '#F8CA1C',
+    'score': 3,
+
+    'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
+                   'requires a user-assisted attack, a small install base, or does very little' +
+                   ' damage.',
+    'banner_required': False
+  },
+
+  'Medium': {
+    'title': 'Medium',
+    'value': 'Medium',
+    'index': 3,
+    'level': 'warning',
+    'color': '#FCA657',
+    'score': 6,
+
+    'description': 'Medium is a real security problem, and is exploitable for many people. ' +
+                   'Includes network daemon denial of service attacks, cross-site scripting, and ' +
+                   'gaining user privileges.',
+    'banner_required': False
+  },
+
+  'High': {
+    'title': 'High',
+    'value': 'High',
+    'index': 2,
+    'level': 'warning',
+    'color': '#F77454',
+    'score': 9,
+
+    'description': 'High is a real problem, exploitable for many people in a default ' +
+                   'installation. Includes serious remote denial of services, local root ' +
+                   'privilege escalations, or data loss.',
+    'banner_required': False
+  },
+
+  'Critical': {
+    'title': 'Critical',
+    'value': 'Critical',
+    'index': 1,
+    'level': 'error',
+    'color': '#D64456',
+    'score': 10,
+
+    'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
+                   'a installation of the package. Includes remote root privilege escalations, ' +
+                   'or massive data loss.',
+    'banner_required': False
+  },
+
+  'Defcon1': {
+    'title': 'Defcon 1',
+    'value': 'Defcon1',
+    'index': 0,
+    'level': 'error',
+    'color': 'black',
+    'score': 11,
+
+    'description': 'Defcon1 is a Critical problem which has been manually highlighted by the Quay' +
+                   ' team. It requires immediate attention.',
+    'banner_required': True
+  }
+}
+
+
+def get_priority_for_index(index):
+  try:
+    int_index = int(index)
+  except ValueError:
+    return 'Unknown'
+
+  for priority in PRIORITY_LEVELS:
+    if PRIORITY_LEVELS[priority]['index'] == int_index:
+      return priority
+
+  return 'Unknown'

util/secscan/analyzer.py

@@ -0,0 +1,205 @@
+import logging
+import logging.config
+
+from collections import defaultdict
+
+import features
+
+from data.database import ExternalNotificationEvent, IMAGE_NOT_SCANNED_ENGINE_VERSION, Image
+from data.model.tag import filter_tags_have_repository_event, get_tags_for_image
+from data.model.image import set_secscan_status, get_image_with_storage_and_parent_base
+from notifications import spawn_notification
+from util.secscan import PRIORITY_LEVELS
+from util.secscan.api import (APIRequestFailure, AnalyzeLayerException, MissingParentLayerException,
+                              InvalidLayerException, AnalyzeLayerRetryException)
+from util.morecollections import AttrDict
+
+
+logger = logging.getLogger(__name__)
+
+
+class PreemptedException(Exception):
+  """ Exception raised if another worker analyzed the image before this worker was able to do so.
+  """
+
+
+class LayerAnalyzer(object):
+  """ Helper class to perform analysis of a layer via the security scanner. """
+  def __init__(self, config, api):
+    self._api = api
+    self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
+
+  def analyze_recursively(self, layer):
+    """ Analyzes a layer and all its parents. Raises a PreemptedException if the analysis was
+        preempted by another worker.
+    """
+    try:
+      self._analyze_recursively_and_check(layer)
+    except MissingParentLayerException:
+      # The parent layer of this layer was missing. Force a reanalyze.
+      try:
+        self._analyze_recursively_and_check(layer, force_parents=True)
+      except MissingParentLayerException:
+        # Parent is still missing... mark the layer as invalid.
+        if not set_secscan_status(layer, False, self._target_version):
+          raise PreemptedException
+
+  def _analyze_recursively_and_check(self, layer, force_parents=False):
+    """ Analyzes a layer and all its parents, optionally forcing parents to be reanalyzed,
+        and checking for various exceptions that can occur during analysis.
+    """
+    try:
+      self._analyze_recursively(layer, force_parents=force_parents)
+    except InvalidLayerException:
+      # One of the parent layers is invalid, so this layer is invalid as well.
+      if not set_secscan_status(layer, False, self._target_version):
+        raise PreemptedException
+    except AnalyzeLayerRetryException:
+      # Something went wrong when trying to analyze the layer, but we should retry, so leave
+      # the layer unindexed. Another worker will come along and handle it.
+      raise APIRequestFailure
+    except MissingParentLayerException:
+      # Pass upward, as missing parent is handled in the analyze_recursively method.
+      raise
+    except AnalyzeLayerException:
+      # Something went wrong when trying to analyze the layer and we cannot retry, so mark the
+      # layer as invalid.
+      logger.exception('Got exception when trying to analyze layer %s via security scanner',
+                       layer.id)
+      if not set_secscan_status(layer, False, self._target_version):
+        raise PreemptedException
+
+  def _analyze_recursively(self, layer, force_parents=False):
+    # Check if there is a parent layer that needs to be analyzed.
+    if layer.parent_id and (force_parents or
+                            layer.parent.security_indexed_engine < self._target_version):
+      try:
+        base_query = get_image_with_storage_and_parent_base()
+        parent_layer = base_query.where(Image.id == layer.parent_id).get()
+      except Image.DoesNotExist:
+        logger.warning("Image %s has Image %s as parent but doesn't exist.", layer.id,
+                       layer.parent_id)
+        raise AnalyzeLayerException('Parent image not found')
+
+      self._analyze_recursively(parent_layer, force_parents=force_parents)
+
+    # Analyze the layer itself.
+    self._analyze(layer, force_parents=force_parents)
+
+  def _analyze(self, layer, force_parents=False):
+    """ Analyzes a single layer.
+
+        Return a tuple of two bools:
+          - The first one tells us if we should evaluate its children.
+          - The second one is set to False when another worker pre-empted the candidate's analysis
+            for us.
+    """
+    # If the parent couldn't be analyzed with the target version or higher, we can't analyze
+    # this image. Mark it as failed with the current target version.
+    if not force_parents and (layer.parent_id and not layer.parent.security_indexed and
+                              layer.parent.security_indexed_engine >= self._target_version):
+      if not set_secscan_status(layer, False, self._target_version):
+        raise PreemptedException
+
+      # Nothing more to do.
+      return
+
+    # Make sure the image's storage is not marked as uploading. If so, nothing more to do.
+    if layer.storage.uploading:
+      if not set_secscan_status(layer, False, self._target_version):
+        raise PreemptedException
+
+      # Nothing more to do.
+      return
+
+    # Analyze the image.
+    previously_security_indexed_successfully = layer.security_indexed
+    previous_security_indexed_engine = layer.security_indexed_engine
+
+    logger.info('Analyzing layer %s', layer.docker_image_id)
+    analyzed_version = self._api.analyze_layer(layer)
+
+    logger.info('Analyzed layer %s successfully with version %s', layer.docker_image_id,
+                analyzed_version)
+
+    # Mark the image as analyzed.
+    if not set_secscan_status(layer, True, analyzed_version):
+      # If the image was previously successfully marked as resolved, then set_secscan_status
+      # might return False because we're not changing it (since this is a fixup).
+      if not previously_security_indexed_successfully:
+        raise PreemptedException
+
+    # If we are the one who've done the job successfully first, then we need to decide if we should
+    # send notifications. Notifications are sent if:
+    #  1) This is a new layer
+    #  2) This is an existing layer that previously did not index properly
+    # We don't always send notifications as if we are re-indexing a successful layer for a newer
+    # feature set in the security scanner, notifications will be spammy.
+    is_new_image = previous_security_indexed_engine == IMAGE_NOT_SCANNED_ENGINE_VERSION
+    is_existing_image_unindexed = not is_new_image and not previously_security_indexed_successfully
+    if (features.SECURITY_NOTIFICATIONS and (is_new_image or is_existing_image_unindexed)):
+      # Get the tags of the layer we analyzed.
+      repository_map = defaultdict(list)
+      event = ExternalNotificationEvent.get(name='vulnerability_found')
+      matching = list(filter_tags_have_repository_event(get_tags_for_image(layer.id), event))
+
+      for tag in matching:
+        repository_map[tag.repository_id].append(tag)
+
+      # If there is at least one tag,
+      # Lookup the vulnerabilities for the image, now that it is analyzed.
+      if len(repository_map) > 0:
+        logger.debug('Loading data for layer %s', layer.id)
+        try:
+          layer_data = self._api.get_layer_data(layer, include_vulnerabilities=True)
+        except APIRequestFailure:
+          raise
+
+        if layer_data is not None:
+          # Dispatch events for any detected vulnerabilities
+          logger.debug('Got data for layer %s: %s', layer.id, layer_data)
+          found_features = layer_data['Layer'].get('Features', [])
+          for repository_id in repository_map:
+            tags = repository_map[repository_id]
+            vulnerabilities = dict()
+
+            # Collect all the vulnerabilities found for the layer under each repository and send
+            # as a batch notification.
+            for feature in found_features:
+              if 'Vulnerabilities' not in feature:
+                continue
+
+              for vulnerability in feature.get('Vulnerabilities', []):
+                vuln_data = {
+                  'id': vulnerability['Name'],
+                  'description': vulnerability.get('Description', None),
+                  'link': vulnerability.get('Link', None),
+                  'has_fix': 'FixedBy' in vulnerability,
+
+                  # TODO: Change this key name if/when we change the event format.
+                  'priority': vulnerability.get('Severity', 'Unknown'),
+                }
+
+                vulnerabilities[vulnerability['Name']] = vuln_data
+
+            # TODO: remove when more endpoints have been converted to using
+            # interfaces
+            repository = AttrDict({
+              'namespace_name': tags[0].repository.namespace_user.username,
+              'name': tags[0].repository.name,
+            })
+
+            repo_vulnerabilities = list(vulnerabilities.values())
+            if not repo_vulnerabilities:
+              continue
+
+            priority_key = lambda v: PRIORITY_LEVELS.get(v['priority'], {}).get('index', 100)
+            repo_vulnerabilities.sort(key=priority_key)
+
+            event_data = {
+              'tags': [tag.name for tag in tags],
+              'vulnerabilities': repo_vulnerabilities,
+              'vulnerability': repo_vulnerabilities[0], # For back-compat with existing events.
+            }
+
+            spawn_notification(repository, 'vulnerability_found', event_data)

util/secscan/api.py

@@ -0,0 +1,503 @@
+import os
+import logging
+
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
+from urlparse import urljoin
+
+import requests
+
+from data import model
+from data.database import CloseForLongOperation, TagManifest, Image, Manifest, ManifestLegacyImage
+from data.model.storage import get_storage_locations
+from data.model.image import get_image_with_storage
+from data.registry_model.datatypes import Manifest as ManifestDataType, LegacyImage
+from util.abchelpers import nooper
+from util.failover import failover, FailoverException
+from util.secscan.validator import SecurityConfigValidator
+from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
+
+from _init import CONF_DIR
+TOKEN_VALIDITY_LIFETIME_S = 60  # Amount of time the security scanner has to call the layer URL
+
+UNKNOWN_PARENT_LAYER_ERROR_MSG = 'worker: parent layer is unknown, it must be processed first'
+
+MITM_CERT_PATH = os.path.join(CONF_DIR, 'mitm.cert')
+
+DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
+
+logger = logging.getLogger(__name__)
+
+
+class AnalyzeLayerException(Exception):
+  """ Exception raised when a layer fails to analyze due to a request issue. """
+
+class AnalyzeLayerRetryException(Exception):
+  """ Exception raised when a layer fails to analyze due to a request issue, and the request should
+      be retried.
+  """
+
+class MissingParentLayerException(AnalyzeLayerException):
+  """ Exception raised when the parent of the layer is missing from the security scanner. """
+
+class InvalidLayerException(AnalyzeLayerException):
+  """ Exception raised when the layer itself cannot be handled by the security scanner. """
+
+class APIRequestFailure(Exception):
+  """ Exception raised when there is a failure to conduct an API request. """
+
+class Non200ResponseException(Exception):
+  """ Exception raised when the upstream API returns a non-200 HTTP status code. """
+  def __init__(self, response):
+    super(Non200ResponseException, self).__init__()
+    self.response = response
+
+
+_API_METHOD_INSERT = 'layers'
+_API_METHOD_GET_LAYER = 'layers/%s'
+_API_METHOD_DELETE_LAYER = 'layers/%s'
+_API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s'
+_API_METHOD_GET_NOTIFICATION = 'notifications/%s'
+_API_METHOD_PING = 'metrics'
+
+
+def compute_layer_id(layer):
+  """ Returns the ID for the layer in the security scanner. """
+  # NOTE: this is temporary until we switch to Clair V3.
+  if isinstance(layer, ManifestDataType):
+    if layer._is_tag_manifest:
+      layer = TagManifest.get(id=layer._db_id).tag.image
+    else:
+      manifest = Manifest.get(id=layer._db_id)
+      try:
+        layer = ManifestLegacyImage.get(manifest=manifest).image
+      except ManifestLegacyImage.DoesNotExist:
+        return None
+  elif isinstance(layer, LegacyImage):
+    layer = Image.get(id=layer._db_id)
+
+  assert layer.docker_image_id
+  assert layer.storage.uuid
+  return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+
+
+class SecurityScannerAPI(object):
+  """ Helper class for talking to the Security Scan service (usually Clair). """
+  def __init__(self, config, storage, server_hostname=None, client=None, skip_validation=False, uri_creator=None, instance_keys=None):
+    feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False)
+    has_valid_config = skip_validation
+
+    if not skip_validation and feature_enabled:
+      config_validator = SecurityConfigValidator(feature_enabled, config.get('SECURITY_SCANNER_ENDPOINT'))
+      has_valid_config = config_validator.valid()
+
+    if feature_enabled and has_valid_config:
+      self.state = ImplementedSecurityScannerAPI(config, storage, server_hostname, client=client, uri_creator=uri_creator, instance_keys=instance_keys)
+    else:
+      self.state = NoopSecurityScannerAPI()
+
+  def __getattr__(self, name):
+    return getattr(self.state, name, None)
+
+
+@add_metaclass(ABCMeta)
+class SecurityScannerAPIInterface(object):
+  """ Helper class for talking to the Security Scan service (usually Clair). """
+  @abstractmethod
+  def cleanup_layers(self, layers):
+    """ Callback invoked by garbage collection to cleanup any layers that no longer
+        need to be stored in the security scanner.
+    """
+    pass
+
+  @abstractmethod
+  def ping(self):
+    """ Calls GET on the metrics endpoint of the security scanner to ensure it is running
+        and properly configured. Returns the HTTP response.
+    """
+    pass
+
+  @abstractmethod
+  def delete_layer(self, layer):
+    """ Calls DELETE on the given layer in the security scanner, removing it from
+        its database.
+    """
+    pass
+
+  @abstractmethod
+  def analyze_layer(self, layer):
+    """ Posts the given layer to the security scanner for analysis, blocking until complete.
+        Returns the analysis version on success or raises an exception deriving from
+        AnalyzeLayerException on failure. Callers should handle all cases of AnalyzeLayerException.
+    """
+    pass
+
+  @abstractmethod
+  def check_layer_vulnerable(self, layer_id, cve_name):
+    """ Checks to see if the layer with the given ID is vulnerable to the specified CVE. """
+    pass
+
+  @abstractmethod
+  def get_notification(self, notification_name, layer_limit=100, page=None):
+    """ Gets the data for a specific notification, with optional page token.
+        Returns a tuple of the data (None on failure) and whether to retry.
+    """
+    pass
+
+  @abstractmethod
+  def mark_notification_read(self, notification_name):
+    """ Marks a security scanner notification as read. """
+    pass
+
+  @abstractmethod
+  def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
+    """ Returns the layer data for the specified layer. On error, returns None. """
+    pass
+
+
+@nooper
+class NoopSecurityScannerAPI(SecurityScannerAPIInterface):
+  """ No-op version of the security scanner API. """
+  pass
+
+
+class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
+  """ Helper class for talking to the Security Scan service (Clair). """
+  # TODO refactor this to not take an app config, and instead just the things it needs as a config object
+  def __init__(self, config, storage, server_hostname, client=None, uri_creator=None, instance_keys=None):
+    self._config = config
+    self._instance_keys = instance_keys
+    self._client = client
+    self._storage = storage
+    self._server_hostname = server_hostname
+    self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
+    self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
+    self._uri_creator = uri_creator
+
+  def _get_image_url_and_auth(self, image):
+    """ Returns a tuple of the url and the auth header value that must be used
+        to fetch the layer data itself. If the image can't be addressed, we return
+        None.
+    """
+    if self._instance_keys is None:
+      raise Exception('No Instance keys provided to Security Scanner API')
+
+    path = model.storage.get_layer_path(image.storage)
+    locations = self._default_storage_locations
+
+    if not self._storage.exists(locations, path):
+      locations = get_storage_locations(image.storage.uuid)
+      if not locations or not self._storage.exists(locations, path):
+        logger.warning('Could not find a valid location to download layer %s out of %s',
+                       compute_layer_id(image), locations)
+        return None, None
+
+    uri = self._storage.get_direct_download_url(locations, path)
+    auth_header = None
+    if uri is None:
+      # Use the registry API instead, with a signed JWT giving access
+      repo_name = image.repository.name
+      namespace_name = image.repository.namespace_user.username
+      repository_and_namespace = '/'.join([namespace_name, repo_name])
+
+      # Generate the JWT which will authorize this
+      audience = self._server_hostname
+      context, subject = build_context_and_subject()
+      access = [{
+        'type': 'repository',
+        'name': repository_and_namespace,
+        'actions': ['pull'],
+      }]
+
+      auth_token = generate_bearer_token(audience, subject, context, access,
+                                         TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
+      auth_header = 'Bearer ' + auth_token
+
+      uri = self._uri_creator(repository_and_namespace, image.storage.content_checksum)
+
+    return uri, auth_header
+
+  def _new_analyze_request(self, layer):
+    """ Create the request body to submit the given layer for analysis. If the layer's URL cannot
+        be found, returns None.
+    """
+    layer_id = compute_layer_id(layer)
+    if layer_id is None:
+      return None
+
+    url, auth_header = self._get_image_url_and_auth(layer)
+    if url is None:
+      return None
+
+    layer_request = {
+      'Name': layer_id,
+      'Path': url,
+      'Format': 'Docker',
+    }
+
+    if auth_header is not None:
+      layer_request['Headers'] = {
+        'Authorization': auth_header,
+      }
+
+    if layer.parent is not None:
+      if layer.parent.docker_image_id and layer.parent.storage.uuid:
+        layer_request['ParentName'] = compute_layer_id(layer.parent)
+
+    return {
+      'Layer': layer_request,
+    }
+
+  def cleanup_layers(self, layers):
+    """ Callback invoked by garbage collection to cleanup any layers that no longer
+        need to be stored in the security scanner.
+    """
+    for layer in layers:
+      self.delete_layer(layer)
+
+  def ping(self):
+    """ Calls GET on the metrics endpoint of the security scanner to ensure it is running
+        and properly configured. Returns the HTTP response.
+    """
+    try:
+      return self._call('GET', _API_METHOD_PING)
+    except requests.exceptions.Timeout as tie:
+      logger.exception('Timeout when trying to connect to security scanner endpoint')
+      msg = 'Timeout when trying to connect to security scanner endpoint: %s' % tie.message
+      raise Exception(msg)
+    except requests.exceptions.ConnectionError as ce:
+      logger.exception('Connection error when trying to connect to security scanner endpoint')
+      msg = 'Connection error when trying to connect to security scanner endpoint: %s' % ce.message
+      raise Exception(msg)
+    except (requests.exceptions.RequestException, ValueError) as ve:
+      logger.exception('Exception when trying to connect to security scanner endpoint')
+      msg = 'Exception when trying to connect to security scanner endpoint: %s' % ve
+      raise Exception(msg)
+
+  def delete_layer(self, layer):
+    """ Calls DELETE on the given layer in the security scanner, removing it from
+        its database.
+    """
+    layer_id = compute_layer_id(layer)
+    if layer_id is None:
+      return None
+
+    # NOTE: We are adding an extra check here for the time being just to be sure we're
+    # not hitting any overlap.
+    docker_image_id, layer_storage_uuid = layer_id.split('.')
+    if get_image_with_storage(docker_image_id, layer_storage_uuid):
+      logger.warning('Found shared Docker ID and storage for layer %s', layer_id)
+      return False
+
+    try:
+      self._call('DELETE', _API_METHOD_DELETE_LAYER % layer_id)
+      return True
+    except Non200ResponseException:
+      return False
+    except requests.exceptions.RequestException:
+      logger.exception('Failed to delete layer: %s', layer_id)
+      return False
+
+  def analyze_layer(self, layer):
+    """ Posts the given layer to the security scanner for analysis, blocking until complete.
+        Returns the analysis version on success or raises an exception deriving from
+        AnalyzeLayerException on failure. Callers should handle all cases of AnalyzeLayerException.
+    """
+    def _response_json(request, response):
+      try:
+        return response.json()
+      except ValueError:
+        logger.exception('Failed to decode JSON when analyzing layer %s', request['Layer']['Name'])
+        raise AnalyzeLayerException
+
+    request = self._new_analyze_request(layer)
+    if not request:
+      logger.error('Could not build analyze request for layer %s', layer.id)
+      raise AnalyzeLayerException
+
+    logger.info('Analyzing layer %s', request['Layer']['Name'])
+    try:
+      response = self._call('POST', _API_METHOD_INSERT, body=request)
+    except requests.exceptions.Timeout:
+      logger.exception('Timeout when trying to post layer data response for %s', layer.id)
+      raise AnalyzeLayerRetryException
+    except requests.exceptions.ConnectionError:
+      logger.exception('Connection error when trying to post layer data response for %s', layer.id)
+      raise AnalyzeLayerRetryException
+    except (requests.exceptions.RequestException) as re:
+      logger.exception('Failed to post layer data response for %s: %s', layer.id, re)
+      raise AnalyzeLayerException
+    except Non200ResponseException as ex:
+      message = _response_json(request, ex.response).get('Error').get('Message', '')
+      logger.warning('A warning event occurred when analyzing layer %s (status code %s): %s',
+                     request['Layer']['Name'], ex.response.status_code, message)
+      # 400 means the layer could not be analyzed due to a bad request.
+      if ex.response.status_code == 400:
+        if message == UNKNOWN_PARENT_LAYER_ERROR_MSG:
+          raise MissingParentLayerException('Bad request to security scanner: %s' % message)
+        else:
+          logger.exception('Got non-200 response for analyze of layer %s', layer.id)
+          raise AnalyzeLayerException('Bad request to security scanner: %s' % message)
+      # 422 means that the layer could not be analyzed:
+      # - the layer could not be extracted (might be a manifest or an invalid .tar.gz)
+      # - the layer operating system / package manager is unsupported
+      elif ex.response.status_code == 422:
+        raise InvalidLayerException
+
+      # Otherwise, it is some other error and we should retry.
+      raise AnalyzeLayerRetryException
+
+    # Return the parsed API version.
+    return _response_json(request, response)['Layer']['IndexedByVersion']
+
+  def check_layer_vulnerable(self, layer_id, cve_name):
+    """ Checks to see if the layer with the given ID is vulnerable to the specified CVE. """
+    layer_data = self._get_layer_data(layer_id, include_vulnerabilities=True)
+    if layer_data is None or 'Layer' not in layer_data or 'Features' not in layer_data['Layer']:
+      return False
+
+    for feature in layer_data['Layer']['Features']:
+      for vuln in feature.get('Vulnerabilities', []):
+        if vuln['Name'] == cve_name:
+          return True
+
+    return False
+
+  def get_notification(self, notification_name, layer_limit=100, page=None):
+    """ Gets the data for a specific notification, with optional page token.
+        Returns a tuple of the data (None on failure) and whether to retry.
+    """
+    try:
+      params = {
+        'limit': layer_limit
+      }
+
+      if page is not None:
+        params['page'] = page
+
+      response = self._call('GET', _API_METHOD_GET_NOTIFICATION % notification_name, params=params)
+      json_response = response.json()
+    except requests.exceptions.Timeout:
+      logger.exception('Timeout when trying to get notification for %s', notification_name)
+      return None, True
+    except requests.exceptions.ConnectionError:
+      logger.exception('Connection error when trying to get notification for %s', notification_name)
+      return None, True
+    except (requests.exceptions.RequestException, ValueError):
+      logger.exception('Failed to get notification for %s', notification_name)
+      return None, False
+    except Non200ResponseException as ex:
+      return None, ex.response.status_code != 404 and ex.response.status_code != 400
+
+    return json_response, False
+
+  def mark_notification_read(self, notification_name):
+    """ Marks a security scanner notification as read. """
+    try:
+      self._call('DELETE', _API_METHOD_MARK_NOTIFICATION_READ % notification_name)
+      return True
+    except Non200ResponseException:
+      return False
+    except requests.exceptions.RequestException:
+      logger.exception('Failed to mark notification as read: %s', notification_name)
+      return False
+
+  def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
+    """ Returns the layer data for the specified layer. On error, returns None. """
+    layer_id = compute_layer_id(layer)
+    if layer_id is None:
+      return None
+
+    return self._get_layer_data(layer_id, include_features, include_vulnerabilities)
+
+  def _get_layer_data(self, layer_id, include_features=False, include_vulnerabilities=False):
+    params = {}
+    if include_features:
+      params = {'features': True}
+
+    if include_vulnerabilities:
+      params = {'vulnerabilities': True}
+
+    try:
+      response = self._call('GET', _API_METHOD_GET_LAYER % layer_id, params=params)
+      logger.debug('Got response %s for vulnerabilities for layer %s',
+                   response.status_code, layer_id)
+      try:
+        return response.json()
+      except ValueError:
+        logger.exception('Failed to decode response JSON')
+        return None
+
+    except Non200ResponseException as ex:
+      logger.debug('Got failed response %s for vulnerabilities for layer %s',
+                   ex.response.status_code, layer_id)
+      if ex.response.status_code == 404:
+        return None
+      else:
+        logger.error(
+          'downstream security service failure: status %d, text: %s',
+          ex.response.status_code,
+          ex.response.text,
+        )
+        if ex.response.status_code // 100 == 5:
+          raise APIRequestFailure('Downstream service returned 5xx')
+        else:
+          raise APIRequestFailure('Downstream service returned non-200')
+    except requests.exceptions.Timeout:
+      logger.exception('API call timed out for loading vulnerabilities for layer %s', layer_id)
+      raise APIRequestFailure('API call timed out')
+    except requests.exceptions.ConnectionError:
+      logger.exception('Connection error for loading vulnerabilities for layer %s', layer_id)
+      raise APIRequestFailure('Could not connect to security service')
+    except requests.exceptions.RequestException:
+      logger.exception('Failed to get layer data response for %s', layer_id)
+      raise APIRequestFailure()
+
+
+  def _request(self, method, endpoint, path, body, params, timeout):
+    """ Issues an HTTP request to the security endpoint. """
+    url = _join_api_url(endpoint, self._config.get('SECURITY_SCANNER_API_VERSION', 'v1'), path)
+    signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8081')
+
+    logger.debug('%sing security URL %s', method.upper(), url)
+    resp = self._client.request(method, url, json=body, params=params, timeout=timeout,
+                                verify=MITM_CERT_PATH, headers=DEFAULT_HTTP_HEADERS,
+                                proxies={'https': 'https://' + signer_proxy_url,
+                                         'http': 'http://' + signer_proxy_url})
+    if resp.status_code // 100 != 2:
+      raise Non200ResponseException(resp)
+    return resp
+
+  def _call(self, method, path, params=None, body=None):
+    """ Issues an HTTP request to the security endpoint handling the logic of using an alternative
+        BATCH endpoint for non-GET requests and failover for GET requests.
+    """
+    timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 1)
+    endpoint = self._config['SECURITY_SCANNER_ENDPOINT']
+
+    with CloseForLongOperation(self._config):
+      # If the request isn't a read, attempt to use a batch stack and do not fail over.
+      if method != 'GET':
+        if self._config.get('SECURITY_SCANNER_ENDPOINT_BATCH') is not None:
+          endpoint = self._config['SECURITY_SCANNER_ENDPOINT_BATCH']
+          timeout = self._config.get('SECURITY_SCANNER_API_BATCH_TIMEOUT_SECONDS') or timeout
+        return self._request(method, endpoint, path, body, params, timeout)
+
+      # The request is read-only and can failover.
+      all_endpoints = [endpoint] + self._config.get('SECURITY_SCANNER_READONLY_FAILOVER_ENDPOINTS', [])
+      return _failover_read_request(*[((self._request, endpoint, path, body, params, timeout), {})
+                                      for endpoint in all_endpoints])
+
+
+def _join_api_url(endpoint, api_version, path):
+  pathless_url = urljoin(endpoint, '/' + api_version) + '/'
+  return urljoin(pathless_url, path)
+
+
+@failover
+def _failover_read_request(request_fn, endpoint, path, body, params, timeout):
+  """ This function auto-retries read-only requests until they return a 2xx status code. """
+  try:
+    return request_fn('GET', endpoint, path, body, params, timeout)
+  except (requests.exceptions.RequestException, Non200ResponseException) as ex:
+    raise FailoverException(ex)

util/secscan/fake.py

@@ -0,0 +1,348 @@
+import json
+import copy
+import uuid
+import urlparse
+
+from contextlib import contextmanager
+from httmock import urlmatch, HTTMock, all_requests
+
+from util.secscan.api import UNKNOWN_PARENT_LAYER_ERROR_MSG, compute_layer_id
+
+@contextmanager
+def fake_security_scanner(hostname='fakesecurityscanner'):
+  """ Context manager which yields a fake security scanner. All requests made to the given
+      hostname (default: fakesecurityscanner) will be handled by the fake.
+  """
+  scanner = FakeSecurityScanner(hostname)
+  with HTTMock(*(scanner.get_endpoints())):
+    yield scanner
+
+
+class FakeSecurityScanner(object):
+  """ Implements a fake security scanner (with somewhat real responses) for testing API calls and
+      responses.
+  """
+  def __init__(self, hostname, index_version=1):
+    self.hostname = hostname
+    self.index_version = index_version
+    self.layers = {}
+    self.notifications = {}
+    self.layer_vulns = {}
+
+    self.ok_layer_id = None
+    self.fail_layer_id = None
+    self.internal_error_layer_id = None
+    self.error_layer_id = None
+    self.unexpected_status_layer_id = None
+
+  def set_ok_layer_id(self, ok_layer_id):
+    """ Sets a layer ID that, if encountered when the analyze call is made, causes a 200
+        to be immediately returned.
+    """
+    self.ok_layer_id = ok_layer_id
+
+  def set_fail_layer_id(self, fail_layer_id):
+    """ Sets a layer ID that, if encountered when the analyze call is made, causes a 422
+        to be raised.
+    """
+    self.fail_layer_id = fail_layer_id
+
+  def set_internal_error_layer_id(self, internal_error_layer_id):
+    """ Sets a layer ID that, if encountered when the analyze call is made, causes a 500
+        to be raised.
+    """
+    self.internal_error_layer_id = internal_error_layer_id
+
+  def set_error_layer_id(self, error_layer_id):
+    """ Sets a layer ID that, if encountered when the analyze call is made, causes a 400
+        to be raised.
+    """
+    self.error_layer_id = error_layer_id
+
+  def set_unexpected_status_layer_id(self, layer_id):
+    """ Sets a layer ID that, if encountered when the analyze call is made, causes an HTTP 600
+        to be raised. This is useful in testing the robustness of the to unknown status codes.
+    """
+    self.unexpected_status_layer_id = layer_id
+
+  def has_layer(self, layer_id):
+    """ Returns true if the layer with the given ID has been analyzed. """
+    return layer_id in self.layers
+
+  def has_notification(self, notification_id):
+    """ Returns whether a notification with the given ID is found in the scanner. """
+    return notification_id in self.notifications
+
+  def add_notification(self, old_layer_ids, new_layer_ids, old_vuln, new_vuln, max_per_page=100,
+                       indexed_old_layer_ids=None, indexed_new_layer_ids=None):
+    """ Adds a new notification over the given sets of layer IDs and vulnerability information,
+        returning the structural data of the notification created.
+    """
+    notification_id = str(uuid.uuid4())
+    if old_vuln is None:
+      old_vuln = dict(new_vuln)
+
+    self.notifications[notification_id] = dict(old_layer_ids=old_layer_ids,
+                                               new_layer_ids=new_layer_ids,
+                                               old_vuln=old_vuln,
+                                               new_vuln=new_vuln,
+                                               max_per_page=max_per_page,
+                                               indexed_old_layer_ids=indexed_old_layer_ids,
+                                               indexed_new_layer_ids=indexed_new_layer_ids)
+
+    return self._get_notification_data(notification_id, 0, 100)
+
+  def layer_id(self, layer):
+    """ Returns the Quay Security Scanner layer ID for the given layer (Image row). """
+    return compute_layer_id(layer)
+
+  def add_layer(self, layer_id):
+    """ Adds a layer to the security scanner, with no features or vulnerabilities. """
+    self.layers[layer_id] = {
+      "Name": layer_id,
+      "Format": "Docker",
+      "IndexedByVersion": self.index_version,
+    }
+
+  def remove_layer(self, layer_id):
+    """ Removes a layer from the security scanner. """
+    self.layers.pop(layer_id, None)
+
+  def set_vulns(self, layer_id, vulns):
+    """ Sets the vulnerabilities for the layer with the given ID to those given. """
+    self.layer_vulns[layer_id] = vulns
+
+    # Since this call may occur before the layer is "anaylzed", we only add the data
+    # to the layer itself if present.
+    if self.layers.get(layer_id):
+      layer = self.layers[layer_id]
+      layer['Features'] = layer.get('Features', [])
+      layer['Features'].append({
+        "Name":  'somefeature',
+        "Namespace": 'somenamespace',
+        "Version": 'someversion',
+        "Vulnerabilities": self.layer_vulns[layer_id],
+      })
+
+  def _get_notification_data(self, notification_id, page, limit):
+    """ Returns the structural data for the notification with the given ID, paginated using
+        the given page and limit. """
+    notification = self.notifications[notification_id]
+    limit = min(limit, notification['max_per_page'])
+
+    notification_data = {
+      "Name": notification_id,
+      "Created": "1456247389",
+      "Notified": "1456246708",
+      "Limit": limit,
+    }
+
+    start_index = (page*limit)
+    end_index = ((page+1)*limit)
+    has_additional_page = False
+
+    if notification.get('old_vuln'):
+      old_layer_ids = notification['old_layer_ids']
+      old_layer_ids = old_layer_ids[start_index:end_index]
+      has_additional_page = has_additional_page or bool(len(old_layer_ids[end_index-1:]))
+
+      notification_data['Old'] = {
+        'Vulnerability': notification['old_vuln'],
+        'LayersIntroducingVulnerability': old_layer_ids,
+      }
+
+      if notification.get('indexed_old_layer_ids', None):
+        indexed_old_layer_ids = notification['indexed_old_layer_ids'][start_index:end_index]
+        notification_data['Old']['OrderedLayersIntroducingVulnerability'] = indexed_old_layer_ids
+
+
+    if notification.get('new_vuln'):
+      new_layer_ids = notification['new_layer_ids']
+      new_layer_ids = new_layer_ids[start_index:end_index]
+      has_additional_page = has_additional_page or bool(len(new_layer_ids[end_index-1:]))
+
+      notification_data['New'] = {
+        'Vulnerability': notification['new_vuln'],
+        'LayersIntroducingVulnerability': new_layer_ids,
+      }
+
+      if notification.get('indexed_new_layer_ids', None):
+        indexed_new_layer_ids = notification['indexed_new_layer_ids'][start_index:end_index]
+        notification_data['New']['OrderedLayersIntroducingVulnerability'] = indexed_new_layer_ids
+
+
+    if has_additional_page:
+      notification_data['NextPage'] = str(page+1)
+
+    return notification_data
+
+  def get_endpoints(self):
+    """ Returns the HTTMock endpoint definitions for the fake security scanner. """
+    @urlmatch(netloc=r'(.*\.)?' + self.hostname, path=r'/v1/layers/(.+)', method='GET')
+    def get_layer_mock(url, request):
+      layer_id = url.path[len('/v1/layers/'):]
+      if layer_id == self.ok_layer_id:
+        return {
+          'status_code': 200,
+          'content': json.dumps({'Layer': {}}),
+        }
+
+      if layer_id == self.internal_error_layer_id:
+        return {
+          'status_code': 500,
+          'content': json.dumps({'Error': {'Message': 'Internal server error'}}),
+        }
+
+      if not layer_id in self.layers:
+        return {
+          'status_code': 404,
+          'content': json.dumps({'Error': {'Message': 'Unknown layer'}}),
+        }
+
+      layer_data = copy.deepcopy(self.layers[layer_id])
+
+      has_vulns = request.url.find('vulnerabilities') > 0
+      has_features = request.url.find('features') > 0
+      if not has_vulns and not has_features:
+        layer_data.pop('Features', None)
+
+      return {
+        'status_code': 200,
+        'content': json.dumps({'Layer': layer_data}),
+      }
+
+    @urlmatch(netloc=r'(.*\.)?' + self.hostname, path=r'/v1/layers/(.+)', method='DELETE')
+    def remove_layer_mock(url, _):
+      layer_id = url.path[len('/v1/layers/'):]
+      if not layer_id in self.layers:
+        return {
+          'status_code': 404,
+          'content': json.dumps({'Error': {'Message': 'Unknown layer'}}),
+        }
+
+      self.layers.pop(layer_id)
+      return {
+        'status_code': 204, 'content': '',
+      }
+
+    @urlmatch(netloc=r'(.*\.)?' + self.hostname, path=r'/v1/layers', method='POST')
+    def post_layer_mock(_, request):
+      body_data = json.loads(request.body)
+      if not 'Layer' in body_data:
+        return {'status_code': 400, 'content': 'Missing body'}
+
+      layer = body_data['Layer']
+      if not 'Path' in layer:
+        return {'status_code': 400, 'content': 'Missing Path'}
+
+      if not 'Name' in layer:
+        return {'status_code': 400, 'content': 'Missing Name'}
+
+      if not 'Format' in layer:
+        return {'status_code': 400, 'content': 'Missing Format'}
+
+      if layer['Name'] == self.internal_error_layer_id:
+        return {
+          'status_code': 500,
+          'content': json.dumps({'Error': {'Message': 'Internal server error'}}),
+        }
+
+      if layer['Name'] == self.fail_layer_id:
+        return {
+          'status_code': 422,
+          'content': json.dumps({'Error': {'Message': 'Cannot analyze'}}),
+        }
+
+      if layer['Name'] == self.error_layer_id:
+        return {
+          'status_code': 400,
+          'content': json.dumps({'Error': {'Message': 'Some sort of error'}}),
+        }
+
+      if layer['Name'] == self.unexpected_status_layer_id:
+        return {
+          'status_code': 600,
+          'content': json.dumps({'Error': {'Message': 'Some sort of error'}}),
+        }
+
+
+      parent_id = layer.get('ParentName', None)
+      parent_layer = None
+
+      if parent_id is not None:
+        parent_layer = self.layers.get(parent_id, None)
+        if parent_layer is None:
+          return {
+            'status_code': 400,
+            'content': json.dumps({'Error': {'Message': UNKNOWN_PARENT_LAYER_ERROR_MSG}}),
+          }
+
+      self.add_layer(layer['Name'])
+      if parent_layer is not None:
+        self.layers[layer['Name']]['ParentName'] = parent_id
+
+      # If vulnerabilities have already been registered with this layer, call set_vulns to make sure
+      # their data is added to the layer's data.
+      if self.layer_vulns.get(layer['Name']):
+        self.set_vulns(layer['Name'], self.layer_vulns[layer['Name']])
+
+      return {
+        'status_code': 201,
+        'content': json.dumps({
+          "Layer": self.layers[layer['Name']],
+        }),
+      }
+
+
+    @urlmatch(netloc=r'(.*\.)?' + self.hostname, path=r'/v1/notifications/(.+)$', method='DELETE')
+    def delete_notification(url, _):
+      notification_id = url.path[len('/v1/notifications/'):]
+      if notification_id not in self.notifications:
+        return {
+          'status_code': 404,
+          'content': json.dumps({'Error': {'Message': 'Unknown notification'}}),
+        }
+
+      self.notifications.pop(notification_id)
+      return {
+        'status_code': 204,
+        'content': '',
+      }
+
+
+    @urlmatch(netloc=r'(.*\.)?' + self.hostname, path=r'/v1/notifications/(.+)$', method='GET')
+    def get_notification(url, _):
+      notification_id = url.path[len('/v1/notifications/'):]
+      if notification_id not in self.notifications:
+        return {
+          'status_code': 404,
+          'content': json.dumps({'Error': {'Message': 'Unknown notification'}}),
+        }
+
+      query_params = urlparse.parse_qs(url.query)
+      limit = int(query_params.get('limit', [2])[0])
+      page = int(query_params.get('page', [0])[0])
+
+      notification_data = self._get_notification_data(notification_id, page, limit)
+      response = {'Notification': notification_data}
+      return {
+        'status_code': 200,
+        'content': json.dumps(response),
+      }
+
+    @urlmatch(netloc=r'(.*\.)?' + self.hostname, path=r'/v1/metrics$', method='GET')
+    def metrics(url, _):
+      return {
+        'status_code': 200,
+        'content': json.dumps({'fake': True}),
+      }
+
+    @all_requests
+    def response_content(url, _):
+      return {
+        'status_code': 500,
+        'content': json.dumps({'Error': {'Message': 'Unknown endpoint %s' % url.path}}),
+      }
+
+    return [get_layer_mock, post_layer_mock, remove_layer_mock, get_notification,
+            delete_notification, metrics, response_content]

util/secscan/notifier.py

@@ -0,0 +1,178 @@
+import logging
+import sys
+
+from collections import defaultdict
+from enum import Enum
+
+from app import secscan_api
+from data.registry_model import registry_model
+from notifications import notification_batch
+from util.secscan import PRIORITY_LEVELS
+from util.secscan.api import APIRequestFailure
+from util.morecollections import AttrDict, StreamingDiffTracker, IndexedStreamingDiffTracker
+
+
+logger = logging.getLogger(__name__)
+
+class ProcessNotificationPageResult(Enum):
+  FINISHED_PAGE = 'Finished Page'
+  FINISHED_PROCESSING = 'Finished Processing'
+  FAILED = 'Failed'
+
+
+class SecurityNotificationHandler(object):
+  """ Class to process paginated notifications from the security scanner and issue
+      Quay vulnerability_found notifications for all necessary tags. Callers should
+      initialize, call process_notification_page_data for each page until it returns
+      FINISHED_PROCESSING or FAILED and, if succeeded, then call send_notifications
+      to send out the notifications queued.
+  """
+  def __init__(self, results_per_stream):
+    self.tags_by_repository_map = defaultdict(set)
+    self.repository_map = {}
+    self.check_map = {}
+    self.layer_ids = set()
+
+    self.stream_tracker = None
+    self.results_per_stream = results_per_stream
+    self.vulnerability_info = None
+
+  def send_notifications(self):
+    """ Sends all queued up notifications. """
+    if self.vulnerability_info is None:
+      return
+
+    new_vuln = self.vulnerability_info
+    new_severity = PRIORITY_LEVELS.get(new_vuln.get('Severity', 'Unknown'), {'index': sys.maxint})
+
+    # For each of the tags found, issue a notification.
+    with notification_batch() as spawn_notification:
+      for repository_id, tags in self.tags_by_repository_map.iteritems():
+        event_data = {
+          'tags': list(tags),
+          'vulnerability': {
+            'id': new_vuln['Name'],
+            'description': new_vuln.get('Description', None),
+            'link': new_vuln.get('Link', None),
+            'priority': new_severity['title'],
+            'has_fix': 'FixedIn' in new_vuln,
+          },
+        }
+
+        spawn_notification(self.repository_map[repository_id], 'vulnerability_found', event_data)
+
+  def process_notification_page_data(self, notification_page_data):
+    """ Processes the given notification page data to spawn vulnerability notifications as
+        necessary. Returns the status of the processing.
+    """
+    if not 'New' in notification_page_data:
+      return self._done()
+
+    new_data = notification_page_data['New']
+    old_data = notification_page_data.get('Old', {})
+
+    new_vuln = new_data['Vulnerability']
+    old_vuln = old_data.get('Vulnerability', {})
+
+    self.vulnerability_info = new_vuln
+
+    new_layer_ids = new_data.get('LayersIntroducingVulnerability', [])
+    old_layer_ids = old_data.get('LayersIntroducingVulnerability', [])
+
+    new_severity = PRIORITY_LEVELS.get(new_vuln.get('Severity', 'Unknown'), {'index': sys.maxint})
+    old_severity = PRIORITY_LEVELS.get(old_vuln.get('Severity', 'Unknown'), {'index': sys.maxint})
+
+    # Check if the severity of the vulnerability has increased. If so, then we report this
+    # vulnerability for *all* layers, rather than a difference, as it is important for everyone.
+    if new_severity['index'] < old_severity['index']:
+      # The vulnerability has had its severity increased. Report for *all* layers.
+      all_layer_ids = set(new_layer_ids) | set(old_layer_ids)
+      for layer_id in all_layer_ids:
+        self._report(layer_id)
+
+      if 'NextPage' not in notification_page_data:
+        return self._done()
+      else:
+        return ProcessNotificationPageResult.FINISHED_PAGE
+
+    # Otherwise, only send the notification to new layers. To find only the new layers, we
+    # need to do a streaming diff vs the old layer IDs stream.
+
+    # Check for ordered data. If found, we use the indexed tracker, which is faster and
+    # more memory efficient.
+    is_indexed = False
+    if ('OrderedLayersIntroducingVulnerability' in new_data or
+        'OrderedLayersIntroducingVulnerability' in old_data):
+
+      def tuplize(stream):
+        return [(entry['LayerName'], entry['Index']) for entry in stream]
+
+      new_layer_ids = tuplize(new_data.get('OrderedLayersIntroducingVulnerability', []))
+      old_layer_ids = tuplize(old_data.get('OrderedLayersIntroducingVulnerability', []))
+      is_indexed = True
+
+    # If this is the first call, initialize the tracker.
+    if self.stream_tracker is None:
+      self.stream_tracker = (IndexedStreamingDiffTracker(self._report, self.results_per_stream)
+                             if is_indexed
+                             else StreamingDiffTracker(self._report, self.results_per_stream))
+
+    # Call to add the old and new layer ID streams to the tracker. The tracker itself will
+    # call _report whenever it has determined a new layer has been found.
+    self.stream_tracker.push_new(new_layer_ids)
+    self.stream_tracker.push_old(old_layer_ids)
+
+    # Check to see if there are any additional pages to process.
+    if 'NextPage' not in notification_page_data:
+      return self._done()
+    else:
+      return ProcessNotificationPageResult.FINISHED_PAGE
+
+  def _done(self):
+    if self.stream_tracker is not None:
+      # Mark the tracker as done, so that it finishes reporting any outstanding layers.
+      self.stream_tracker.done()
+
+    # Process all the layers.
+    if self.vulnerability_info is not None:
+      if not self._process_layers():
+        return ProcessNotificationPageResult.FAILED
+
+    return ProcessNotificationPageResult.FINISHED_PROCESSING
+
+  def _report(self, new_layer_id):
+    self.layer_ids.add(new_layer_id)
+
+  def _chunk(self, pairs, chunk_size):
+    start_index = 0
+    while start_index < len(pairs):
+      yield pairs[start_index:chunk_size]
+      start_index += chunk_size
+
+  def _process_layers(self):
+    cve_id = self.vulnerability_info['Name']
+
+    # Builds the pairs of layer ID and storage uuid.
+    pairs = [tuple(layer_id.split('.', 2)) for layer_id in self.layer_ids]
+
+    # Find the matching tags.
+    for current_pairs in self._chunk(pairs, 50):
+      tags = list(registry_model.yield_tags_for_vulnerability_notification(current_pairs))
+      for tag in tags:
+        # Verify that the tag's *top layer* has the vulnerability.
+        if not tag.layer_id in self.check_map:
+          logger.debug('Checking if layer %s is vulnerable to %s', tag.layer_id, cve_id)
+          try:
+            self.check_map[tag.layer_id] = secscan_api.check_layer_vulnerable(tag.layer_id, cve_id)
+          except APIRequestFailure:
+            return False
+
+        logger.debug('Result of layer %s is vulnerable to %s check: %s', tag.layer_id, cve_id,
+                     self.check_map[tag.layer_id])
+
+        if self.check_map[tag.layer_id]:
+          # Add the vulnerable tag to the list.
+          self.tags_by_repository_map[tag.repository.id].add(tag.name)
+          self.repository_map[tag.repository.id] = tag.repository
+
+    return True

util/secscan/secscan_util.py

@@ -0,0 +1,22 @@
+from urlparse import urljoin
+
+from flask import url_for
+
+
+def get_blob_download_uri_getter(context, url_scheme_and_hostname):
+  """
+  Returns a function with context to later generate the uri for a download blob
+  :param context: Flask RequestContext
+  :param url_scheme_and_hostname: URLSchemeAndHostname class instance
+  :return: function (repository_and_namespace, checksum) -> uri
+  """
+  def create_uri(repository_and_namespace, checksum):
+    """
+    Creates a uri for a download blob from a repository, namespace, and checksum from earlier context
+    """
+    with context:
+      relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
+                                   digest=checksum)
+    return urljoin(url_scheme_and_hostname.get_url(), relative_layer_url)
+
+  return create_uri

util/secscan/test/test_secscan_util.py

@@ -0,0 +1,19 @@
+import pytest
+
+from app import app
+from util.config import URLSchemeAndHostname
+from util.secscan.secscan_util import get_blob_download_uri_getter
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('url_scheme_and_hostname, repo_namespace, checksum, expected_value,', [
+  (URLSchemeAndHostname('http', 'localhost:5000'),
+   'devtable/simple', 'tarsum+sha256:123',
+   'http://localhost:5000/v2/devtable/simple/blobs/tarsum%2Bsha256:123'),
+])
+def test_blob_download_uri_getter(app, url_scheme_and_hostname,
+                                         repo_namespace, checksum,
+                                         expected_value):
+  blob_uri_getter = get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname)
+
+  assert blob_uri_getter(repo_namespace, checksum) == expected_value

util/secscan/validator.py

@@ -0,0 +1,30 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class SecurityConfigValidator(object):
+  """ Helper class for validating the security scanner configuration. """
+  def __init__(self, feature_sec_scan, sec_scan_endpoint):
+    if not feature_sec_scan:
+      return
+
+    self._feature_sec_scan = feature_sec_scan
+    self._sec_scan_endpoint = sec_scan_endpoint
+
+  def valid(self):
+    if not self._feature_sec_scan:
+      return False
+
+    if self._sec_scan_endpoint is None:
+      logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
+      return False
+
+    endpoint = self._sec_scan_endpoint
+    if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
+      logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
+      return False
+
+    return True
+