Merge remote-tracking branch 'upstream/master' into python-registry-v2

app.py

@@ -37,7 +37,7 @@ from util.saas.metricqueue import MetricQueue
 from util.config.provider import get_config_provider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
-from util.secscan.secscanendpoint import SecurityScanEndpoint
+from util.secscan.api import SecurityScannerAPI
 
 OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
 OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
@@ -160,7 +160,9 @@ image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf)
 dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
                                    reporter=MetricQueueReporter(metric_queue))
 notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
-secscan_endpoint = SecurityScanEndpoint(app, config_provider)
+secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf)
+
+secscan_api = SecurityScannerAPI(app, config_provider)
 
 # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
 _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)

conf/init/service/security_notification_worker/log/run

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec logger -i -t securitynotificationworker

conf/init/service/security_notification_worker/run

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+echo 'Starting security scanner notification worker'
+
+cd /
+venv/bin/python -m workers.security_notification_worker 2>&1
+
+echo 'Security scanner notification worker exited'

config.py

@@ -50,7 +50,7 @@ class DefaultConfig(object):
 
   CONTACT_INFO = [
     'mailto:support@quay.io',
-    'irc://chat.freenode.net:6665/quayio',
+    'irc://chat.freenode.net:6665/quay',
     'tel:+1-888-930-3475',
     'https://twitter.com/quayio',
   ]
@@ -130,6 +130,7 @@ class DefaultConfig(object):
   DIFFS_QUEUE_NAME = 'imagediff'
   DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild'
   REPLICATION_QUEUE_NAME = 'imagestoragereplication'
+  SECSCAN_NOTIFICATION_QUEUE_NAME = 'secscan_notification'
 
   # Super user config. Note: This MUST BE an empty list for the default config.
   SUPER_USERS = []
@@ -257,7 +258,7 @@ class DefaultConfig(object):
   # Security scanner
   FEATURE_SECURITY_SCANNER = False
   SECURITY_SCANNER = {
-    'ENDPOINT': 'http://192.168.99.100:6060',
+    'ENDPOINT': 'http://192.168.99.101:6060',
     'ENGINE_VERSION_TARGET': 1,
     'API_VERSION': 'v1',
     'API_TIMEOUT_SECONDS': 10,

data/migrations/versions/50925110da8c_add_event_specific_config.py

@@ -14,7 +14,6 @@ from alembic import op
 import sqlalchemy as sa
 from util.migrate import UTF8LongText
 
-
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     op.add_column('repositorynotification', sa.Column('event_config_json', UTF8LongText, nullable=False))

data/model/image.py

@@ -1,7 +1,7 @@
 import logging
 import dateutil.parser
 
-from peewee import JOIN_LEFT_OUTER, fn
+from peewee import JOIN_LEFT_OUTER, fn, SQL
 from datetime import datetime
 
 from data.model import (DataModelException, db_transaction, _basequery, storage,
@@ -13,6 +13,25 @@ from data.database import (Image, Repository, ImageStoragePlacement, Namespace,
 logger = logging.getLogger(__name__)
 
 
+def get_repository_image_and_deriving(docker_image_id, storage_uuid):
+  """ Returns all matching images with the given docker image ID and storage uuid, along with any
+      images which have the image ID as parents.
+  """
+  try:
+    image_found = (Image
+                    .select()
+                    .join(ImageStorage)
+                    .where(Image.docker_image_id == docker_image_id,
+                           ImageStorage.uuid == storage_uuid)
+                    .get())
+  except Image.DoesNotExist:
+    return Image.select().where(Image.id < 0) # Empty query
+
+  ancestors_pattern = '%s%s/%%' % (image_found.ancestors, image_found.id)
+  return Image.select().where((Image.ancestors ** ancestors_pattern) |
+                              (Image.id == image_found.id))
+
+
 def get_parent_images(namespace_name, repository_name, image_obj):
   """ Returns a list of parent Image objects starting with the most recent parent
       and ending with the base layer.

data/model/tag.py

@@ -13,6 +13,20 @@ def _tag_alive(query, now_ts=None):
                      (RepositoryTag.lifetime_end_ts > now_ts))
 
 
+def get_matching_tags(docker_image_id, storage_uuid, *args):
+  """ Returns a query pointing to all tags that contain the image with the
+      given docker_image_id and storage_uuid. """
+  image_query = image.get_repository_image_and_deriving(docker_image_id, storage_uuid)
+
+  return (RepositoryTag
+            .select(*args)
+            .distinct()
+            .join(Image)
+            .join(ImageStorage)
+            .where(Image.id << image_query, RepositoryTag.lifetime_end_ts >> None,
+                   RepositoryTag.hidden == False))
+
+
 def list_repository_tags(namespace_name, repository_name, include_hidden=False,
                          include_storage=False):
   to_select = (RepositoryTag, Image)

endpoints/api/repositorynotification.py

@@ -22,12 +22,19 @@ def notification_view(note):
   except:
     config = {}
 
+  event_config = {}
+  try:
+    event_config = json.loads(note.event_config_json)
+  except:
+    event_config = {}
+
   return {
     'uuid': note.uuid,
     'event': note.event.name,
     'method': note.method.name,
     'config': config,
     'title': note.title,
+    'event_config': event_config,
   }
 
 
@@ -160,7 +167,7 @@ class TestRepositoryNotification(RepositoryParamResource):
       raise NotFound()
 
     event_info = NotificationEvent.get_event(test_note.event.name)
-    sample_data = event_info.get_sample_data(repository=test_note.repository)
+    sample_data = event_info.get_sample_data(test_note)
     notification_data = build_notification_data(test_note, sample_data)
     notification_queue.put([test_note.repository.namespace_user.username, repository,
                             test_note.event.name], json.dumps(notification_data))

endpoints/api/secscan.py

@@ -5,7 +5,7 @@ import features
 import json
 import requests
 
-from app import secscan_endpoint
+from app import secscan_api
 from data import model
 from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
                            RepositoryParamResource, resource, nickname, show_if, parse_args,
@@ -15,10 +15,17 @@ from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_pa
 logger = logging.getLogger(__name__)
 
 
+class SCAN_STATUS(object):
+  """ Security scan status enum """
+  SCANNED = 'scanned'
+  FAILED = 'failed'
+  QUEUED = 'queued'
+
+
 def _call_security_api(relative_url, *args, **kwargs):
   """ Issues an HTTP call to the sec API at the given relative URL. """
   try:
-    response = secscan_endpoint.call_api(relative_url, *args, **kwargs)
+    response = secscan_api.call(relative_url, None, *args, **kwargs)
   except requests.exceptions.Timeout:
     raise DownstreamIssue(payload=dict(message='API call timed out'))
   except requests.exceptions.ConnectionError:
@@ -39,37 +46,44 @@ def _call_security_api(relative_url, *args, **kwargs):
   return response_data
 
 
+def _get_status(repo_image):
+  if repo_image.security_indexed_engine is not None and repo_image.security_indexed_engine >= 0:
+    return SCAN_STATUS.SCANNED if repo_image.security_indexed else SCAN_STATUS.FAILED
+
+  return SCAN_STATUS.QUEUED
+
+
 @show_if(features.SECURITY_SCANNER)
-@resource('/v1/repository/<repopath:repository>/tag/<tag>/vulnerabilities')
+@resource('/v1/repository/<repopath:repository>/image/<imageid>/vulnerabilities')
 @path_param('repository', 'The full path of the repository. e.g. namespace/name')
-@path_param('tag', 'The name of the tag')
+@path_param('imageid', 'The image ID')
-class RepositoryTagVulnerabilities(RepositoryParamResource):
+class RepositoryImageVulnerabilities(RepositoryParamResource):
-  """ Operations for managing the vulnerabilities in a repository tag. """
+  """ Operations for managing the vulnerabilities in a repository image. """
 
   @require_repo_read
-  @nickname('getRepoTagVulnerabilities')
+  @nickname('getRepoImageVulnerabilities')
   @parse_args
   @query_param('minimumPriority', 'Minimum vulnerability priority', type=str,
                default='Low')
-  def get(self, args, namespace, repository, tag):
+  def get(self, args, namespace, repository, imageid):
     """ Fetches the vulnerabilities (if any) for a repository tag. """
-    try:
+    repo_image = model.image.get_repo_image(namespace, repository, imageid)
-      tag_image = model.tag.get_tag_image(namespace, repository, tag)
+    if repo_image is None:
-    except model.DataModelException:
       raise NotFound()
 
-    if not tag_image.security_indexed:
+    if not repo_image.security_indexed:
-      logger.debug('Image %s for tag %s under repository %s/%s not security indexed',
+      logger.debug('Image %s under repository %s/%s not security indexed',
-                   tag_image.docker_image_id, tag, namespace, repository)
+                   repo_image.docker_image_id, namespace, repository)
       return {
-        'security_indexed': False
+        'status': _get_status(repo_image),
       }
 
-    data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id,
+    layer_id = '%s.%s' % (repo_image.docker_image_id, repo_image.storage.uuid)
+    data = _call_security_api('layers/%s/vulnerabilities', layer_id,
                               minimumPriority=args.minimumPriority)
 
     return {
-      'security_indexed': True,
+      'status': _get_status(repo_image),
       'data': data,
     }
 
@@ -91,13 +105,14 @@ class RepositoryImagePackages(RepositoryParamResource):
 
     if not repo_image.security_indexed:
       return {
-        'security_indexed': False
+        'status': _get_status(repo_image),
       }
 
-    data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id)
+    layer_id = '%s.%s' % (repo_image.docker_image_id, repo_image.storage.uuid)
+    data = _call_security_api('layers/%s/packages', layer_id)
 
     return {
-      'security_indexed': True,
+      'status': _get_status(repo_image),
       'data': data,
     }
 

endpoints/common.py

@@ -22,6 +22,7 @@ from werkzeug.routing import BaseConverter
 from functools import wraps
 from config import frontend_visible_config
 from external_libraries import get_external_javascript, get_external_css
+from util.secscan.api import PRIORITY_LEVELS
 
 import features
 
@@ -183,6 +184,7 @@ def render_page_template(name, **kwargs):
                                        config_set=json.dumps(frontend_visible_config(app.config)),
                                        oauth_set=json.dumps(get_oauth_config()),
                                        scope_set=json.dumps(scopes.app_scopes(app.config)),
+                                       vuln_priority_set=json.dumps(PRIORITY_LEVELS),
                                        mixpanel_key=app.config.get('MIXPANEL_KEY', ''),
                                        google_analytics_key=app.config.get('GOOGLE_ANALYTICS_KEY', ''),
                                        sentry_public_dsn=app.config.get('SENTRY_PUBLIC_DSN', ''),

endpoints/notificationevent.py

@@ -1,9 +1,11 @@
 import logging
 import time
+import json
 
 from datetime import datetime
 from notificationhelper import build_event_data
 from util.jinjautil import get_template_env
+from util.secscan.api import PRIORITY_LEVELS, get_priority_for_index
 
 template_env = get_template_env("events")
 logger = logging.getLogger(__name__)
@@ -37,13 +39,18 @@ class NotificationEvent(object):
       'notification_data': notification_data
     })
 
-  def get_sample_data(self, repository=None):
+  def get_sample_data(self, notification):
     """
-    Returns sample data for testing the raising of this notification, with an optional
+    Returns sample data for testing the raising of this notification, with an example notification.
-    repository.
     """
     raise NotImplementedError
 
+  def should_perform(self, event_data, notification_data):
+    """
+    Whether a notification for this event should be performed. By default returns True.
+    """
+    return True
+
   @classmethod
   def event_name(cls):
     """
@@ -71,8 +78,8 @@ class RepoPushEvent(NotificationEvent):
   def get_summary(self, event_data, notification_data):
     return 'Repository %s updated' % (event_data['repository'])
 
-  def get_sample_data(self, repository):
+  def get_sample_data(self, notification):
-    return build_event_data(repository, {
+    return build_event_data(notification.repository, {
       'updated_tags': {'latest': 'someimageid', 'foo': 'anotherimage'},
       'pruned_image_count': 3
     })
@@ -99,18 +106,27 @@ class VulnerabilityFoundEvent(NotificationEvent):
 
     return 'info'
 
-  def get_sample_data(self, repository):
+  def get_sample_data(self, notification):
-    return build_event_data(repository, {
+    event_config = json.loads(notification.event_config_json)
+
+    return build_event_data(notification.repository, {
       'tags': ['latest', 'prod'],
       'image': 'some-image-id',
       'vulnerability': {
         'id': 'CVE-FAKE-CVE',
         'description': 'A futurist vulnerability',
         'link': 'https://security-tracker.debian.org/tracker/CVE-FAKE-CVE',
-        'priority': 'Critical',
+        'priority': get_priority_for_index(event_config['level'])
       },
     })
 
+  def should_perform(self, event_data, notification_data):
+    event_config = json.loads(notification_data.event_config_json)
+    expected_level_index = event_config['level']
+    priority = PRIORITY_LEVELS[event_data['vulnerability']['priority']]
+    actual_level_index = priority['index']
+    return expected_level_index <= actual_level_index
+
   def get_summary(self, event_data, notification_data):
     msg = '%s vulnerability detected in repository %s in tags %s'
     return msg % (event_data['vulnerability']['priority'],
@@ -126,10 +142,10 @@ class BuildQueueEvent(NotificationEvent):
   def get_level(self, event_data, notification_data):
     return 'info'
 
-  def get_sample_data(self, repository):
+  def get_sample_data(self, notification):
     build_uuid = 'fake-build-id'
 
-    return build_event_data(repository, {
+    return build_event_data(notification.repository, {
       'is_manual': False,
       'build_id': build_uuid,
       'build_name': 'some-fake-build',
@@ -165,10 +181,10 @@ class BuildStartEvent(NotificationEvent):
   def get_level(self, event_data, notification_data):
     return 'info'
 
-  def get_sample_data(self, repository):
+  def get_sample_data(self, notification):
     build_uuid = 'fake-build-id'
 
-    return build_event_data(repository, {
+    return build_event_data(notification.repository, {
       'build_id': build_uuid,
       'build_name': 'some-fake-build',
       'docker_tags': ['latest', 'foo', 'bar'],
@@ -193,10 +209,10 @@ class BuildSuccessEvent(NotificationEvent):
   def get_level(self, event_data, notification_data):
     return 'success'
 
-  def get_sample_data(self, repository):
+  def get_sample_data(self, notification):
     build_uuid = 'fake-build-id'
 
-    return build_event_data(repository, {
+    return build_event_data(notification.repository, {
       'build_id': build_uuid,
       'build_name': 'some-fake-build',
       'docker_tags': ['latest', 'foo', 'bar'],
@@ -222,10 +238,10 @@ class BuildFailureEvent(NotificationEvent):
   def get_level(self, event_data, notification_data):
     return 'error'
 
-  def get_sample_data(self, repository):
+  def get_sample_data(self, notification):
     build_uuid = 'fake-build-id'
 
-    return build_event_data(repository, {
+    return build_event_data(notification.repository, {
       'build_id': build_uuid,
       'build_name': 'some-fake-build',
       'docker_tags': ['latest', 'foo', 'bar'],

endpoints/secscan.py

@@ -0,0 +1,25 @@
+import logging
+import json
+
+import features
+
+from app import secscan_notification_queue
+from flask import request, make_response, Blueprint
+from endpoints.common import route_show_if
+
+logger = logging.getLogger(__name__)
+secscan = Blueprint('secscan', __name__)
+
+@route_show_if(features.SECURITY_SCANNER)
+@secscan.route('/notification', methods=['POST'])
+def secscan_notification():
+  data = request.get_json()
+  logger.debug('Got notification from Clair: %s', data)
+
+  content = data['Content']
+  layer_ids = content.get('NewIntroducingLayersIDs', content.get('IntroducingLayersIDs', []))
+  if not layer_ids:
+    return make_response('Okay')
+
+  secscan_notification_queue.put(['notification', data['Name']], json.dumps(data))
+  return make_response('Okay')

endpoints/v1/registry.py

@@ -7,7 +7,7 @@ from functools import wraps
 from datetime import datetime
 from time import time
 
-from app import storage as store, image_diff_queue, image_replication_queue, app
+from app import storage as store, image_replication_queue, app
 from auth.auth import process_auth, extract_namespace_repo_from_session
 from auth.auth_context import get_authenticated_user, get_grant_user_context
 from digest import checksums
@@ -41,20 +41,9 @@ def _finish_image(namespace, repository, repo_image):
   # Checksum is ok, we remove the marker
   set_uploading_flag(repo_image, False)
 
-  image_id = repo_image.docker_image_id
-
-  # The layer is ready for download, send a job to the work queue to
-  # process it.
-  logger.debug('Adding layer to diff queue')
-  repo = model.repository.get_repository(namespace, repository)
-  image_diff_queue.put([repo.namespace_user.username, repository, image_id], json.dumps({
-    'namespace_user_id': repo.namespace_user.id,
-    'repository': repository,
-    'image_id': image_id,
-  }))
-
   # Send a job to the work queue to replicate the image layer.
   if features.STORAGE_REPLICATION:
+    repo = model.repository.get_repository(namespace, repository)
     image_replication_queue.put([repo_image.storage.uuid], json.dumps({
       'namespace_user_id': repo.namespace_user.id,
       'storage_id': repo_image.storage.uuid,

pylintrc

@@ -9,7 +9,7 @@
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=missing-docstring
+disable=missing-docstring,invalid-name,too-many-locals
 
 [TYPECHECK]
 

static/css/directives/repo-view/repo-panel-info.css

@@ -3,10 +3,56 @@
   float: right;
 }
 
+.repo-panel-info-element .right-sec-controls {
+  border: 1px solid #ddd;
+  padding: 20px;
+  border-radius: 4px;
+  max-width: 400px;
+}
+
+.repo-panel-info-element .right-sec-controls {
+  color: #333;
+  font-weight: 300;
+  padding-left: 70px;
+  position: relative;
+}
+
+.repo-panel-info-element .right-sec-controls .sec-logo {
+  position: absolute;
+  top: 17px;
+  left: 15px;
+}
+
+.repo-panel-info-element .right-sec-controls .sec-logo .lock {
+  position: absolute;
+  top: 5px;
+  right: 10px;
+}
+
+.repo-panel-info-element .right-sec-controls b {
+  color: #333;
+  font-weight: normal;
+  margin-bottom: 20px;
+  display: block;
+}
+
+.repo-panel-info-element .right-sec-controls .configure-alerts {
+  margin-top: 20px;
+  font-weight: normal;
+}
+
+.repo-panel-info-element .right-sec-controls .configure-alerts .fa {
+  margin-right: 6px;
+}
+
+.repo-panel-info-element .right-sec-controls .repository-events-summary {
+  margin-top: 20px;
+}
+
 .repo-panel-info-element .right-controls .copy-box {
   width: 400px;
-  display: inline-block;
+  margin-top: 10px;
-  margin-left: 10px;
+  margin-bottom: 20px;
 }
 
 .repo-panel-info-element .stat-col {

static/css/directives/repo-view/repo-panel-tags.css

@@ -85,6 +85,37 @@
   margin-right: 2px;
 }
 
+.repo-panel-tags-element .security-scan-col span {
+  cursor: pointer;
+}
+
+.repo-panel-tags-element .security-scan-col i.fa {
+  margin-right: 4px;
+}
+
+.repo-panel-tags-element .security-scan-col .scanning,
+.repo-panel-tags-element .security-scan-col .failed-scan,
+.repo-panel-tags-element .security-scan-col .vuln-load-error {
+  color: #9B9B9B;
+  font-size: 12px;
+}
+
+.repo-panel-tags-element .security-scan-col .no-vulns a {
+  color: #2FC98E;
+}
+
+.repo-panel-tags-element .security-scan-col .vuln-link,
+.repo-panel-tags-element .security-scan-col .vuln-link span {
+  text-decoration: none !important
+}
+
+.repo-panel-tags-element .security-scan-col .has-vulns.Critical .highest-vuln,
+.repo-panel-tags-element .security-scan-col .has-vulns.Defcon1 .highest-vuln {
+}
+
+.repo-panel-tags-element .other-vulns {
+  color: black;
+}
 
 @media (max-width: 767px) {
   .repo-panel-tags-element .tag-span {

static/css/directives/ui/filter-box.css

@@ -16,3 +16,34 @@
   margin-bottom: 10px;
   color: #ccc;
 }
+
+.filter-box.floating {
+  float: right;
+  min-width: 300px;
+  margin-top: 15px;
+  position: relative;
+}
+
+.filter-box.floating .filter-message {
+  position: absolute;
+  left: -200px;
+  top: 7px;
+}
+
+
+@media (max-width: 767px) {
+  .filter-box.floating {
+    float: none;
+    width: 100%;
+    display: block;
+    margin-top: 10px;
+  }
+
+  .filter-box.floating .form-control {
+    max-width: 100%;
+  }
+
+  .filter-box.floating .filter-message {
+    display: none;
+  }
+}

static/css/directives/ui/repository-events-summary.css

@@ -0,0 +1,21 @@
+.repository-events-summary-element .summary-list {
+    padding: 0px;
+    margin: 0px;
+    list-style: none;
+}
+
+.repository-events-summary-element .summary-list li {
+    margin-bottom: 6px;
+}
+
+.repository-events-summary-element .summary-list li i.fa {
+    margin-right: 4px;
+}
+
+.repository-events-summary-element .notification-event-fields {
+    display: inline-block;
+    padding: 0px;
+    margin: 0px;
+    list-style: none;
+    padding-left: 24px;
+}

static/css/directives/ui/repository-events-table.css

@@ -1,3 +1,13 @@
 .repository-events-table-element .notification-row i.fa {
   margin-right: 6px;
 }
+
+.repository-events-table-element .notification-event-fields {
+  list-style: none;
+  padding: 0px;
+  margin-left: 28px;
+  margin-top: 3px;
+  font-size: 13px;
+  color: #888;
+  margin-bottom: 0px;
+}

static/css/directives/ui/vulnerability-priority-view.css

@@ -0,0 +1,19 @@
+.vulnerability-priority-view-element i.fa {
+  margin-right: 4px;
+}
+
+.vulnerability-priority-view-element.Unknown,
+.vulnerability-priority-view-element.Low,
+.vulnerability-priority-view-element.Negligible {
+  color: #9B9B9B;
+}
+
+.vulnerability-priority-view-element.Medium {
+  color: #FCA657;
+}
+
+.vulnerability-priority-view-element.High,
+.vulnerability-priority-view-element.Critical,
+.vulnerability-priority-view-element.Defcon1 {
+  color: #D64456;
+}

static/css/pages/image-view.css

@@ -21,5 +21,50 @@
 }
 
 .image-view .co-tab-content h3 {
+  margin-bottom: 30px;
+}
+
+.image-view .fa-bug {
+  margin-right: 4px;
+}
+
+.image-view .co-filter-box {
+  float: right;
+  min-width: 300px;
+  margin-bottom: 10px;
+}
+
+.image-view .co-filter-box .current-filtered {
+  display: inline-block;
+  margin-right: 10px;
+  color: #999;
+}
+
+.image-view .co-filter-box input {
+  display: inline-block;
+}
+
+.image-view .level-col h4 {
+  margin-top: 0px;
   margin-bottom: 20px;
 }
+
+.image-view .levels {
+  list-style: none;
+  padding: 0px;
+  margin: 0px;
+}
+
+.image-view .levels li {
+  margin-bottom: 20px;
+}
+
+.image-view .levels li .description {
+  margin-top: 6px;
+  font-size: 14px;
+  color: #999;
+}
+
+.image-view .level-col {
+  padding: 20px;
+}

static/directives/header-bar.html

@@ -24,14 +24,16 @@
           <li><a ng-href="/tour/" target="{{ appLinkTarget() }}" quay-section="tour">Tour</a></li>
           <li><a ng-href="/tutorial/" target="{{ appLinkTarget() }}" quay-section="tutorial">Tutorial</a></li>
           <li quay-require="['BILLING']"><a ng-href="/plans/" target="{{ appLinkTarget() }}" quay-section="plans">Pricing</a></li>
-          <li><a href="http://docs.quay.io/" target="_blank">Docs</a></li>
+          <li><a href="https://docs.quay.io/" target="_blank">Docs</a></li>
+          <li><a href="https://blog.quay.io/" target="_blank">Blog</a></li>
         </ul>
 
         <!-- Signed in -->
         <ul class="nav navbar-nav navbar-links" ng-if="!user.anonymous">
           <li><a ng-href="/repository/" target="{{ appLinkTarget() }}" quay-section="repository">Repositories</a></li>
           <li><a ng-href="/tutorial/" target="{{ appLinkTarget() }}" quay-section="tutorial">Tutorial</a></li>
-          <li><a href="http://docs.quay.io/" target="_blank">Docs</a></li>
+          <li><a href="https://docs.quay.io/" target="_blank">Docs</a></li>
+          <li><a href="https://blog.quay.io/" target="_blank">Blog</a></li>
         </ul>
 
         <!-- Phone -->

static/directives/repo-view/repo-panel-info.html

@@ -65,7 +65,25 @@
 
     <!-- Pull Controls -->
     <div class="right-controls hidden-sm hidden-xs">
-      Pull Full Repository: <div class="copy-box" hovering-message="true" value="pullCommand"></div>
+      <div class="right-pull-controls">
+        <div>Pull this container with the following Docker command:</div>
+        <div class="copy-box" hovering-message="true" value="pullCommand"></div>
+      </div>
+
+      <div class="right-sec-controls" quay-show="repository.can_admin && Features.SECURITY_SCANNER">
+        <span class="sec-logo">
+          <img class="lock" src="/static/img/lock.svg">
+          <img class="scan" src="/static/img/scan.svg">
+        </span>
+
+        <b>Automated Security Scanning (Preview)</b>
+        <div>Continually scanning this repository for 17K+ known vulnerabilities. <a href="https://blog.quay.io/security-scanning-beta" target="_blank">Read more about this feature</a>.</div>
+        <div class="configure-alerts" ng-if="!hasEvents">
+          <a href="/repository/{{ repository.namespace }}/{{ repository.name }}?tab=settings&add_event=vulnerability_found"><i class="fa fa-bell-o"></i>Configure Vulnerability Alerts</a>
+        </div>
+        <div class="repository-events-summary" is-enabled="repository.can_admin" repository="repository"
+             event-filter="vulnerability_found" has-events="hasEvents"></div>
+      </div>
     </div>
 
     <h4 style="font-size:20px;">Description</h4>

static/directives/repo-view/repo-panel-tags.html

@@ -81,6 +81,12 @@
           style="min-width: 120px;">
         <a href="javascript:void(0)" ng-click="orderBy('last_modified_datetime')">Last Modified</a>
       </td>
+      <td class="hidden-xs"
+          ng-class="tablePredicateClass('security_scanned', options.predicate, options.reverse)"
+          style="min-width: 120px;"
+          quay-require="['SECURITY_SCANNER']">
+          Security Scan
+      </td>
       <td class="hidden-xs"
           ng-class="tablePredicateClass('size', options.predicate, options.reverse)"
           style="min-width: 62px;">
@@ -107,6 +113,61 @@
         <span am-time-ago="tag.last_modified" bo-if="tag.last_modified"></span>
         <span bo-if="!tag.last_modified">Unknown</span>
       </td>
+      <td quay-require="['SECURITY_SCANNER']" class="security-scan-col">
+        <span class="cor-loader-inline" ng-if="getTagVulnerabilities(tag).loading"></span>
+        <span class="vuln-load-error" ng-if="getTagVulnerabilities(tag).hasError">
+          Could not load security information
+        </span>
+        <span ng-if="!getTagVulnerabilities(tag).loading">
+          <!-- Queued -->
+          <span class="scanning" ng-if="getTagVulnerabilities(tag).status == 'queued'"
+                data-title="The image for this tag is queued to be scanned for vulnerabilities"
+                bs-tooltip>Queued for scan</span>
+
+          <!-- Scan Failed -->
+          <span class="failed-scan" ng-if="getTagVulnerabilities(tag).status == 'failed'"
+                data-title="The image for this tag could not be scanned for vulnerabilities"
+                bs-tooltip>
+            <i class="fa fa-times-circle"></i>
+            Failed to scan
+          </span>
+
+          <!-- No Vulns -->
+          <span class="no-vulns"
+                ng-if="getTagVulnerabilities(tag).status == 'scanned' && !getTagVulnerabilities(tag).hasVulnerabilities"
+                data-title="The image for this tag has no vulnerabilities as found in our database"
+                bs-tooltip
+                bindonce>
+            <a bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=security">
+              <i class="fa fa-check-circle"></i>
+              Passed
+            </a>
+          </span>
+
+          <!-- Vulns -->
+          <span ng-if="getTagVulnerabilities(tag).status == 'scanned' && getTagVulnerabilities(tag).hasVulnerabilities"
+                ng-class="getTagVulnerabilities(tag).highestVulnerability.Priority"
+                class="has-vulns" bindonce>
+              <a class="vuln-link" bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=security"
+                data-title="The image for this tag has {{ getTagVulnerabilities(tag).highestVulnerability.Count }} {{ getTagVulnerabilities(tag).highestVulnerability.Priority }} level vulnerabilities"
+                bs-tooltip>
+              <span class="highest-vuln">
+                <span class="vulnerability-priority-view" priority="getTagVulnerabilities(tag).highestVulnerability.Priority">
+                  {{ getTagVulnerabilities(tag).highestVulnerability.Count }}
+                </span>
+              </span>
+
+              <span ng-if="getTagVulnerabilities(tag).vulnerabilities.length - getTagVulnerabilities(tag).highestVulnerability.Count > 0"
+                    class="other-vulns">
+              + {{ getTagVulnerabilities(tag).vulnerabilities.length - getTagVulnerabilities(tag).highestVulnerability.Count }} others
+              </span>
+            </a>
+            <a bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=security" style="display: inline-block; margin-left: 6px;">
+              More Info
+            </a>
+          </span>
+        </span>
+      </td>
       <td class="hidden-xs" bo-text="tag.size | bytes"></td>
       <td class="hidden-xs image-id-col">
         <span class="image-link" repository="repository" image-id="tag.image_id"></span>

static/directives/repository-events-summary.html

@@ -0,0 +1,22 @@
+<div class="repository-events-summary-element">
+  <div class="resource-view" resource="notificationsResource"
+       error-message="'Could not load repository events'">
+       <ul class="summary-list">
+         <li ng-repeat="notification in notifications">
+            <i class="fa fa-lg" ng-class="getMethodInfo(notification).icon"></i>
+            {{ getMethodInfo(notification).title }} for
+
+            <ul class="notification-event-fields" ng-if="getEventInfo(notification).fields.length">
+              <li ng-repeat="field in getEventInfo(notification).fields">
+                {{ field.title }} of
+                <span ng-switch on="field.type">
+                  <span ng-switch-when="enum">
+                   {{ findEnumValue(field.values, notification.event_config[field.name]).title }}
+                  </span>
+                </span>
+              </li>
+            </ul>
+         </li>
+       </ul>
+    </div>
+</div>

static/directives/repository-events-table.html

@@ -44,6 +44,17 @@
                     <i class="fa fa-lg" ng-class="getEventInfo(notification).icon"></i>
                     {{ getEventInfo(notification).title }}
                   </span>
+
+                  <ul class="notification-event-fields" ng-if="getEventInfo(notification).fields.length">
+                    <li ng-repeat="field in getEventInfo(notification).fields">
+                      {{ field.title }}:
+                      <span ng-switch on="field.type">
+                        <span ng-switch-when="enum">
+                         {{ findEnumValue(field.values, notification.event_config[field.name]).title }}
+                        </span>
+                      </span>
+                    </li>
+                  </ul>
                 </td>
 
                 <td>
@@ -89,5 +100,6 @@
   <div class="create-external-notification-dialog"
        repository="repository"
        counter="showNewNotificationCounter"
+       default-data="newNotificationData"
        notification-created="handleNotificationCreated(notification)"></div>
 </div>

static/directives/vulnerability-priority-view.html

@@ -0,0 +1,5 @@
+<span class="vulnerability-priority-view-element" ng-class="priority">
+  <i class="fa fa-exclamation-triangle"></i>
+  <span ng-transclude/>
+  {{ priority }}
+</span>

static/img/lock.svg

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg width="13px" height="19px" viewBox="0 0 13 19" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
+    <!-- Generator: Sketch 3.4.1 (15681) - http://www.bohemiancoding.com/sketch -->
+    <title>Artboard 1</title>
+    <desc>Created with Sketch.</desc>
+    <defs></defs>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd" sketch:type="MSPage">
+        <g id="Artboard-1" sketch:type="MSArtboardGroup" stroke-width="2" stroke="#1D3447">
+            <g id="Oval-121-+-Rectangle-443" sketch:type="MSLayerGroup" transform="translate(0.000000, 1.000000)">
+                <g sketch:type="MSShapeGroup">
+                    <path d="M9.75,3.41251359 C9.75,1.5540428 8.29492544,0.0474545455 6.5,0.0474545455 C4.70507456,0.0474545455 3.25,1.5540428 3.25,3.41251359 L3.25,7.41109091 L9.75,7.41109091 L9.75,3.41251359 Z" id="Oval-121"></path>
+                    <rect id="Rectangle-443" fill="#2FC98E" x="0" y="5.77472727" width="13" height="11.4545455" rx="2"></rect>
+                </g>
+            </g>
+        </g>
+    </g>
+</svg>

static/img/scan.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 17.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 width="33.223px" height="31px" viewBox="0 0 33.223 31" enable-background="new 0 0 33.223 31" xml:space="preserve">
+<polygon fill-rule="evenodd" clip-rule="evenodd" fill="#1E3547" points="31.311,16.034 33.223,9.604 27.315,10.646 "/>
+<g>
+	<path fill="#1E3547" d="M3.552,11.75c1.638-5.762,6.937-10,13.217-10s11.579,4.238,13.217,10h1.805C30.107,5.013,24.021,0,16.769,0
+		S3.43,5.013,1.747,11.75H3.552z"/>
+	<path fill="#1E3547" d="M30.06,18.96c-1.54,5.909-6.907,10.29-13.292,10.29S5.018,24.87,3.477,18.96H1.672
+		C3.25,25.845,9.413,31,16.769,31s13.519-5.155,15.097-12.04H30.06z"/>
+</g>
+<polygon fill-rule="evenodd" clip-rule="evenodd" fill="#1E3547" points="1.913,14.434 0,20.864 5.909,19.822 "/>
+</svg>

static/js/directives/repo-view/repo-panel-info.js

@@ -10,7 +10,8 @@ angular.module('quay').directive('repoPanelInfo', function () {
     restrict: 'C',
     scope: {
       'repository': '=repository',
-      'builds': '=builds'
+      'builds': '=builds',
+      'isEnabled': '=isEnabled'
     },
     controller: function($scope, $element, ApiService, Config) {
       $scope.$watch('repository', function(repository) {

static/js/directives/repo-view/repo-panel-tags.js

@@ -18,7 +18,7 @@ angular.module('quay').directive('repoPanelTags', function () {
 
       'getImages': '&getImages'
     },
-    controller: function($scope, $element, $filter, $location, ApiService, UIService) {
+    controller: function($scope, $element, $filter, $location, ApiService, UIService, VulnerabilityService) {
       var orderBy = $filter('orderBy');
 
       $scope.checkedTags = UIService.createCheckStateController([], 'name');
@@ -34,7 +34,8 @@ angular.module('quay').directive('repoPanelTags', function () {
       $scope.tagHistory = {};
       $scope.tagActionHandler = null;
       $scope.showingHistory = false;
-      $scope.tagsPerPage = 50;
+      $scope.tagsPerPage = 25;
+      $scope.imageVulnerabilities = {};
 
       var setTagState = function() {
         if (!$scope.repository || !$scope.selectedTags) { return; }
@@ -149,6 +150,68 @@ angular.module('quay').directive('repoPanelTags', function () {
         setTagState();
       });
 
+      $scope.loadImageVulnerabilities = function(image_id, imageData) {
+        var params = {
+          'imageid': image_id,
+          'repository': $scope.repository.namespace + '/' + $scope.repository.name,
+        };
+
+        ApiService.getRepoImageVulnerabilities(null, params).then(function(resp) {
+          imageData.loading = false;
+          imageData.status = resp['status'];
+
+          if (imageData.status == 'scanned') {
+            var vulnerabilities = resp.data.Vulnerabilities;
+
+            imageData.hasVulnerabilities = !!vulnerabilities.length;
+            imageData.vulnerabilities = vulnerabilities;
+
+            var highest = {
+              'Priority': 'Unknown',
+              'Count': 0,
+              'index': 100000
+            };
+
+            resp.data.Vulnerabilities.forEach(function(v) {
+              if (VulnerabilityService.LEVELS[v.Priority].index < highest.index) {
+                highest = {
+                  'Priority': v.Priority,
+                  'Count': 1,
+                  'index': VulnerabilityService.LEVELS[v.Priority].index
+                }
+              } else if (VulnerabilityService.LEVELS[v.Priority].index == highest.index) {
+                highest['Count']++;
+              }
+            });
+
+            imageData.highestVulnerability = highest;
+          }
+        }, function() {
+          imageData.loading = false;
+          imageData.hasError = true;
+        });
+      };
+
+      $scope.getTagVulnerabilities = function(tag) {
+        return $scope.getImageVulnerabilities(tag.image_id);
+      };
+
+      $scope.getImageVulnerabilities = function(image_id) {
+        if (!$scope.repository) {
+          return
+        }
+
+        if (!$scope.imageVulnerabilities[image_id]) {
+          $scope.imageVulnerabilities[image_id] = {
+            'loading': true
+          };
+
+          $scope.loadImageVulnerabilities(image_id, $scope.imageVulnerabilities[image_id]);
+        }
+
+        return $scope.imageVulnerabilities[image_id];
+      };
+
       $scope.clearSelectedTags = function() {
         $scope.checkedTags.setChecked([]);
       };

static/js/directives/ui/create-external-notification-dialog.js

@@ -11,7 +11,8 @@ angular.module('quay').directive('createExternalNotificationDialog', function ()
     scope: {
       'repository': '=repository',
       'counter': '=counter',
-      'notificationCreated': '&notificationCreated'
+      'notificationCreated': '&notificationCreated',
+      'defaultData': '=defaultData'
     },
     controller: function($scope, $element, ExternalNotificationData, ApiService, $timeout, StringBuilderService) {
       $scope.currentEvent = null;
@@ -98,6 +99,13 @@ angular.module('quay').directive('createExternalNotificationDialog', function ()
         ApiService.createRepoNotification(data, params).then(function(resp) {
           $scope.status = '';
           $scope.notificationCreated({'notification': resp});
+
+          // Used by repository-events-summary.
+          if (!$scope.repository._notificationCounter) {
+            $scope.repository._notificationCounter = 0;
+          }
+
+          $scope.repository._notificationCounter++;
           $('#createNotificationModal').modal('hide');
         });
       };
@@ -154,6 +162,13 @@ angular.module('quay').directive('createExternalNotificationDialog', function ()
           $scope.currentEvent = null;
           $scope.currentMethod = null;
           $scope.unauthorizedEmail = false;
+
+          $timeout(function() {
+            if ($scope.defaultData && $scope.defaultData['currentEvent']) {
+              $scope.setEvent($scope.defaultData['currentEvent']);
+            }
+          }, 100);
+
           $('#createNotificationModal').modal({});
         }
       });

static/js/directives/ui/repository-events-summary.js

@@ -0,0 +1,77 @@
+/**
+ * An element which displays a summary of events on a repository of a particular type.
+ */
+angular.module('quay').directive('repositoryEventsSummary', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/repository-events-summary.html',
+    replace: false,
+    transclude: false,
+    restrict: 'C',
+    scope: {
+      'repository': '=repository',
+      'isEnabled': '=isEnabled',
+      'eventFilter': '@eventFilter',
+      'hasEvents': '=hasEvents'
+    },
+    controller: function($scope, ApiService, ExternalNotificationData) {
+      var loadNotifications = function() {
+        if (!$scope.repository || !$scope.isEnabled || !$scope.eventFilter || $scope.notificationsResource) {
+          return;
+        }
+
+        var params = {
+          'repository': $scope.repository.namespace + '/' + $scope.repository.name
+        };
+
+        $scope.notificationsResource = ApiService.listRepoNotificationsAsResource(params).get(
+          function(resp) {
+            var notifications = [];
+            resp.notifications.forEach(function(notification) {
+              if (notification.event == $scope.eventFilter) {
+                notifications.push(notification);
+              }
+            });
+
+            $scope.notifications = notifications;
+            $scope.hasEvents = !!notifications.length;
+            return $scope.notifications;
+          });
+      };
+
+      $scope.$watch('repository', loadNotifications);
+      $scope.$watch('isEnabled', loadNotifications);
+      $scope.$watch('eventFilter', loadNotifications);
+
+      // Watch _notificationCounter, which is set by create-external-notification-dialog. We use this
+      // to invalidate and reload.
+      $scope.$watch('repository._notificationCounter', function() {
+        $scope.notificationsResource = null;
+        loadNotifications();
+      });
+
+      loadNotifications();
+
+      $scope.findEnumValue = function(values, index) {
+        var found = null;
+        Object.keys(values).forEach(function(key) {
+          if (values[key]['index'] == index) {
+            found = values[key];
+            return
+          }
+        });
+
+        return found
+      };
+
+      $scope.getEventInfo = function(notification) {
+        return ExternalNotificationData.getEventInfo(notification.event);
+      };
+
+      $scope.getMethodInfo = function(notification) {
+        return ExternalNotificationData.getMethodInfo(notification.method);
+      };
+    }
+  };
+  return directiveDefinitionObject;
+});

static/js/directives/ui/repository-events-table.js

@@ -13,11 +13,29 @@ angular.module('quay').directive('repositoryEventsTable', function () {
       'repository': '=repository',
       'isEnabled': '=isEnabled'
     },
-    controller: function($scope, $element, ApiService, Restangular, UtilService, ExternalNotificationData) {
+    controller: function($scope, $element, $timeout, ApiService, Restangular, UtilService, ExternalNotificationData, $location) {
       $scope.showNewNotificationCounter = 0;
+      $scope.newNotificationData = {};
 
       var loadNotifications = function() {
-        if (!$scope.repository || $scope.notificationsResource || !$scope.isEnabled) { return; }
+        if (!$scope.repository || !$scope.isEnabled) { return; }
+
+        var add_event = $location.search()['add_event'];
+        if (add_event) {
+          $timeout(function() {
+            $scope.newNotificationData = {
+              'currentEvent': ExternalNotificationData.getEventInfo(add_event)
+            };
+
+            $scope.askCreateNotification();
+          }, 100);
+
+          $location.search('add_event', null);
+        }
+
+        if ($scope.notificationsResource) {
+          return;
+        }
 
         var params = {
           'repository': $scope.repository.namespace + '/' + $scope.repository.name
@@ -43,6 +61,18 @@ angular.module('quay').directive('repositoryEventsTable', function () {
         $scope.showNewNotificationCounter++;
       };
 
+      $scope.findEnumValue = function(values, index) {
+        var found = null;
+        Object.keys(values).forEach(function(key) {
+          if (values[key]['index'] == index) {
+            found = values[key];
+            return
+          }
+        });
+
+        return found
+      };
+
       $scope.getEventInfo = function(notification) {
         return ExternalNotificationData.getEventInfo(notification.event);
       };
@@ -61,6 +91,13 @@ angular.module('quay').directive('repositoryEventsTable', function () {
           var index = $.inArray(notification, $scope.notifications);
           if (index < 0) { return; }
           $scope.notifications.splice(index, 1);
+
+          if (!$scope.repository._notificationCounter) {
+            $scope.repository._notificationCounter = 0;
+          }
+
+          $scope.repository._notificationCounter++;
+
         }, ApiService.errorDisplay('Cannot delete notification'));
       };
 

static/js/directives/ui/vulnerability-priority-view.js

@@ -0,0 +1,18 @@
+/**
+ * An element which displays a priority triangle for vulnerabilities.
+ */
+angular.module('quay').directive('vulnerabilityPriorityView', function () {
+  var directiveDefinitionObject = {
+    priority: 0,
+    templateUrl: '/static/directives/vulnerability-priority-view.html',
+    replace: false,
+    transclude: true,
+    restrict: 'C',
+    scope: {
+      'priority': '=priority'
+    },
+    controller: function($scope, $element) {
+    }
+  };
+  return directiveDefinitionObject;
+});

static/js/pages/image-view.js

@@ -10,11 +10,16 @@
     })
   }]);
 
-  function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService) {
+  function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService, VulnerabilityService, Features) {
     var namespace = $routeParams.namespace;
     var name = $routeParams.name;
     var imageid = $routeParams.image;
 
+    $scope.options = {
+      'vulnFilter': '',
+      'packageFilter': ''
+    };
+
     var loadImage = function() {
       var params = {
         'repository': namespace + '/' + name,
@@ -40,6 +45,46 @@
     loadImage();
     loadRepository();
 
+    $scope.downloadPackages = function() {
+      if (!Features.SECURITY_SCANNER || $scope.packagesResource) { return; }
+
+      var params = {
+        'repository': namespace + '/' + name,
+        'imageid': imageid
+      };
+
+      $scope.packagesResource = ApiService.getRepoImagePackagesAsResource(params).get(function(packages) {
+        $scope.packages = packages;
+        return packages;
+      });
+    };
+
+    $scope.loadImageVulnerabilities = function() {
+      if (!Features.SECURITY_SCANNER || $scope.vulnerabilitiesResource) { return; }
+
+      $scope.VulnerabilityLevels = VulnerabilityService.getLevels();
+
+      var params = {
+        'repository': namespace + '/' + name,
+        'imageid': imageid
+      };
+
+      $scope.vulnerabilitiesResource = ApiService.getRepoImageVulnerabilitiesAsResource(params).get(function(resp) {
+        $scope.vulnerabilityInfo = resp;
+        $scope.vulnerabilities = [];
+
+        if (resp.data && resp.data.Vulnerabilities) {
+          resp.data.Vulnerabilities.forEach(function(vuln) {
+            vuln_copy = jQuery.extend({}, vuln);
+            vuln_copy['index'] = VulnerabilityService.LEVELS[vuln['Priority']]['index'];
+            $scope.vulnerabilities.push(vuln_copy);
+          });
+        }
+
+        return resp;
+      });
+    };
+
     $scope.downloadChanges = function() {
       if ($scope.changesResource) { return; }
 

static/js/pages/repo-view.js

@@ -17,6 +17,7 @@
     var imageLoader = ImageLoaderService.getLoader($scope.namespace, $scope.name);
 
     // Tab-enabled counters.
+    $scope.infoShown = 0;
     $scope.tagsShown = 0;
     $scope.logsShown = 0;
     $scope.buildsShown = 0;
@@ -119,6 +120,10 @@
       $scope.viewScope.selectedTags = $.unique(tagNames.split(','));
     };
 
+    $scope.showInfo = function() {
+      $scope.infoShown++;
+    };
+
     $scope.showBuilds = function() {
       $scope.buildsShown++;
     };

static/js/services/external-notification-data.js

@@ -47,12 +47,12 @@ function(Config, Features, VulnerabilityService) {
     events.push({
       'id': 'vulnerability_found',
       'title': 'Package Vulnerability Found',
-      'icon': 'fa-flag',
+      'icon': 'fa-bug',
       'fields': [
         {
           'name': 'level',
           'type': 'enum',
-          'title': 'Minimum Severity Level',
+          'title': 'Minimum Priority Level',
           'values': VulnerabilityService.LEVELS,
         }
       ]

static/js/services/notification-service.js

@@ -129,7 +129,8 @@ function($rootScope, $interval, UserService, ApiService, StringBuilderService, P
       'message': 'A {vulnerability.priority} vulnerability was detected in repository {repository}',
       'page': function(metadata) {
         return '/repository/' + metadata.repository + '?tab=tags';
-      }
+      },
+      'dismissable': true
     }
   };
 

static/js/services/vulnerability-service.js

@@ -3,89 +3,7 @@
   */
 angular.module('quay').factory('VulnerabilityService', ['Config', function(Config) {
   var vulnService = {};
-
+  vulnService.LEVELS = window.__vuln_priority;
-  // NOTE: This objects are used directly in the external-notification-data service, so make sure
-  // to update that code if the format here is changed.
-  vulnService.LEVELS = {
-    'Unknown': {
-      'title': 'Unknown',
-      'index': '6',
-      'level': 'info',
-
-      'description': 'Unknown is either a security problem that has not been assigned ' +
-                     'to a priority yet or a priority that our system did not recognize',
-      'banner_required': false
-    },
-
-    'Negligible': {
-      'title': 'Negligible',
-      'index': '5',
-      'level': 'info',
-
-      'description': 'Negligible is technically a security problem, but is only theoretical ' +
-                     'in nature, requires a very special situation, has almost no install base, ' +
-                     'or does no real damage.',
-      'banner_required': false
-    },
-
-    'Low': {
-      'title': 'Low',
-      'index': '4',
-      'level': 'warning',
-
-      'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
-                     'requires a user-assisted attack, a small install base, or does very ' +
-                     'little damage.',
-      'banner_required': false
-    },
-
-    'Medium': {
-      'title': 'Medium',
-      'value': 'Medium',
-      'index': '3',
-      'level': 'warning',
-
-      'description': 'Medium is a real security problem, and is exploitable for many people. ' +
-                     'Includes network daemon denial of service attacks, cross-site scripting, ' +
-                     'and gaining user privileges.',
-      'banner_required': false
-    },
-
-    'High': {
-      'title': 'High',
-      'value': 'High',
-      'index': '2',
-      'level': 'warning',
-
-      'description': 'High is a real problem, exploitable for many people in a default installation. ' +
-                     'Includes serious remote denial of services, local root privilege escalations, ' +
-                     'or data loss.',
-      'banner_required': false
-    },
-
-    'Critical': {
-      'title': 'Critical',
-      'value': 'Critical',
-      'index': '1',
-      'level': 'error',
-
-      'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
-                     'a installation of the package. Includes remote root privilege escalations, ' +
-                     'or massive data loss.',
-      'banner_required': true
-    },
-
-    'Defcon1': {
-      'title': 'Defcon 1',
-      'value': 'Defcon1',
-      'index': '0',
-      'level': 'error',
-
-      'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +
-                     'by the Quay team. It requires immediate attention.',
-      'banner_required': true
-    }
-  };
 
   vulnService.getLevels = function() {
     return Object.keys(vulnService.LEVELS).map(function(key) {

static/partials/image-view.html

@@ -25,6 +25,16 @@
               tab-init="downloadChanges()">
           <i class="fa fa-code-fork"></i>
         </span>
+        <span class="cor-tab" tab-title="Security Scan" tab-target="#security"
+              tab-init="loadImageVulnerabilities()"
+              quay-show="Features.SECURITY_SCANNER">
+          <i class="fa fa-bug"></i>
+        </span>
+        <span class="cor-tab" tab-title="Packages" tab-target="#packages"
+              tab-init="downloadPackages()"
+              quay-show="Features.SECURITY_SCANNER">
+          <i class="fa ci-package"></i>
+        </span>
       </div> <!-- /cor-tabs -->
 
       <div class="cor-tab-content">
@@ -53,6 +63,118 @@
             </div>
           </div>
         </div>
+
+        <!-- Security -->
+        <div id="security" class="tab-pane" quay-require="['SECURITY_SCANNER']">
+          <div class="resource-view" resource="vulnerabilitiesResource" error-message="'Could not load security information for image'">
+            <div class="col-md-9">
+              <div class="filter-box floating" collection="vulnerabilities" filter-model="options.vulnFilter" filter-name="Vulnerabilities" ng-if="vulnerabilityInfo.status == 'scanned' && vulnerabilities.length"></div>
+
+              <h3>Image Security</h3>
+              <div class="empty" ng-if="vulnerabilityInfo.status == 'queued'">
+                <div class="empty-primary-msg">This image has not been indexed yet</div>
+                <div class="empty-secondary-msg">
+                  Please try again in a few minutes.
+                </div>
+              </div>
+
+              <div class="empty" ng-if="vulnerabilityInfo.status == 'failed'">
+                <div class="empty-primary-msg">This image could not be indexed</div>
+                <div class="empty-secondary-msg">
+                  Our security scanner was unable to index this image.
+                </div>
+              </div>
+
+              <div class="empty" ng-if="vulnerabilityInfo.status == 'scanned' && !vulnerabilities.length">
+                <div class="empty-primary-msg">This image contains no recognized security vulnerabilities</div>
+                <div class="empty-secondary-msg">
+                  Quay currently indexes Debian, Red Hat and Ubuntu packages.
+                </div>
+              </div>
+
+              <div ng-if="vulnerabilityInfo.status == 'scanned' && vulnerabilities.length">
+                <table class="co-table">
+                  <thead>
+                    <td style="width: 200px;">Vulnerability</td>
+                    <td style="width: 200px;">Priority</td>
+                    <td>Description</td>
+                  </thead>
+
+                  <tr ng-repeat="vulnerability in vulnerabilities | filter:options.vulnFilter | orderBy:'index'">
+                    <td><a href="{{ vulnerability.Link }}" target="_blank">{{ vulnerability.ID }}</a></td>
+                    <td>
+                      <span class="vulnerability-priority-view" priority="vulnerability.Priority"></span>
+                    <td>{{ vulnerability.Description }}</td>
+                  </tr>
+                </table>
+
+                <div class="empty" ng-if="(vulnerabilities | filter:options.vulnFilter).length == 0"
+                     style="margin-top: 20px;">
+                  <div class="empty-primary-msg">No matching vulnerabilities found</div>
+                  <div class="empty-secondary-msg">
+                    Please adjust your filter above.
+                  </div>
+                </div>
+              </div>
+            </div>
+
+            <div class="level-col col-md-3 hidden-sm hidden-xs">
+              <h4>Priority Guide</h4>
+              <ul class="levels">
+                <li ng-repeat="level in VulnerabilityLevels | orderBy: 'index'">
+                  <div class="vulnerability-priority-view" priority="level.title"></div>
+                  <div class="description">
+                    {{ level.description }}
+                  </div>
+                </li>
+              </ul>
+            </div>
+          </div>
+        </div>
+
+        <!-- Packages -->
+        <div id="packages" class="tab-pane" quay-require="['SECURITY_SCANNER']">
+          <div class="resource-view" resource="packagesResource" error-message="'Could not load image packages'">
+            <div class="filter-box floating" collection="packages.data.Packages" filter-model="options.packageFilter" filter-name="Packages" ng-if="packages.status == 'scanned' && packages.data.Packages.length"></div>
+
+            <h3>Image Packages</h3>
+            <div class="empty" ng-if="packages.status == 'queued'">
+              <div class="empty-primary-msg">This image has not been indexed yet</div>
+              <div class="empty-secondary-msg">
+                Please try again in a few minutes.
+              </div>
+            </div>
+
+            <div class="empty" ng-if="packages.status == 'failed'">
+              <div class="empty-primary-msg">This image could not be indexed</div>
+              <div class="empty-secondary-msg">
+                Our security scanner was unable to index this image.
+              </div>
+            </div>
+
+            <table class="co-table" ng-if="packages.status == 'scanned'">
+              <thead>
+                <td>Package Name</td>
+                <td>Package Version</td>
+                <td>OS</td>
+              </thead>
+
+              <tr ng-repeat="package in packages.data.Packages | filter:options.packageFilter | orderBy:'Name'">
+                <td>{{ package.Name }}</td>
+                <td>{{ package.Version }}</td>
+                <td>{{ package.OS }}</td>
+              </tr>
+            </table>
+
+            <div class="empty" ng-if="(packages.data.Packages | filter:options.packageFilter).length == 0"
+                   style="margin-top: 20px;">
+                <div class="empty-primary-msg">No matching packages found</div>
+                <div class="empty-secondary-msg" ng-if="options.packageFilter">
+                  Please adjust your filter above.
+                </div>
+              </div>
+          </div>
+        </div>
      </div>
     </div>
   </div>

static/partials/repo-view.html

@@ -17,7 +17,8 @@
 
     <div class="cor-tab-panel">
       <div class="cor-tabs">
-        <span class="cor-tab" tab-active="true" tab-title="Information" tab-target="#info">
+        <span class="cor-tab" tab-active="true" tab-title="Information" tab-target="#info"
+              tab-init="showInfo()">
           <i class="fa fa-info-circle"></i>
         </span>
 
@@ -56,7 +57,8 @@
         <div id="info" class="tab-pane active">
           <div class="repo-panel-info"
                repository="viewScope.repository"
-               builds="viewScope.builds"></div>
+               builds="viewScope.builds"
+               is-enabled="infoShown"></div>
         </div>
 
         <!-- Tags -->

templates/base.html

@@ -38,6 +38,7 @@
       window.__config = {{ config_set|safe }};
       window.__oauth = {{ oauth_set|safe }};
       window.__auth_scopes = {{ scope_set|safe }};
+      window.__vuln_priority = {{ vuln_priority_set|safe }}
       window.__token = '{{ csrf_token() }}';
     </script>
 
@@ -106,7 +107,6 @@ mixpanel.init("{{ mixpanel_key }}", { track_pageview : false, debug: {{ is_debug
       <div class="col-md-8">
         <ul>
           <li><span class="copyright">&copy;2015 CoreOS, Inc.</span></li>
-          <li><a href="http://blog.quay.io">Blog</a></li>
           <li quay-require="['BILLING']"><a href="/tos" target="_self">Terms</a></li>
           <li quay-require="['BILLING']"><a href="/privacy" target="_self">Privacy</a></li>
           <li quay-require="['BILLING']"><a href="/security/" target="_self">Security</a></li>

test/test_api_security.py

@@ -49,8 +49,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
                                      SuperUserSendRecoveryEmail, ChangeLog,
                                      SuperUserOrganizationManagement, SuperUserOrganizationList,
                                      SuperUserAggregateLogs)
-
+from endpoints.api.secscan import RepositoryImagePackages, RepositoryImageVulnerabilities
-from endpoints.api.secscan import RepositoryImagePackages, RepositoryTagVulnerabilities
 
 
 try:
@@ -4225,10 +4224,10 @@ class TestOrganizationInvoiceField(ApiTestCase):
     self._run_test('DELETE', 201, 'devtable', None)
 
 
-class TestRepositoryTagVulnerabilities(ApiTestCase):
+class TestRepositoryImageVulnerabilities(ApiTestCase):
   def setUp(self):
     ApiTestCase.setUp(self)
-    self._set_url(RepositoryTagVulnerabilities, repository='devtable/simple', tag='latest')
+    self._set_url(RepositoryImageVulnerabilities, repository='devtable/simple', imageid='fake')
 
   def test_get_anonymous(self):
     self._run_test('GET', 401, None, None)
@@ -4240,7 +4239,7 @@ class TestRepositoryTagVulnerabilities(ApiTestCase):
     self._run_test('GET', 403, 'reader', None)
 
   def test_get_devtable(self):
-    self._run_test('GET', 200, 'devtable', None)
+    self._run_test('GET', 404, 'devtable', None)
 
 
 class TestRepositoryImagePackages(ApiTestCase):

util/migrate/allocator.py

@@ -126,6 +126,7 @@ def yield_random_entries(batch_query, primary_key_field, batch_size, max_id):
       an "id" field which can be inspected.
   """
 
+  max_id = max(max_id, 1)
   allocator = CompletedKeys(max_id + 1)
 
   try:

util/secscan/api.py

@@ -0,0 +1,224 @@
+import features
+import logging
+import requests
+
+from data.database import CloseForLongOperation
+from urlparse import urljoin
+
+logger = logging.getLogger(__name__)
+
+# NOTE: This objects are used directly in the external-notification-data and vulnerability-service
+# on the frontend, so be careful with changing their existing keys.
+PRIORITY_LEVELS = {
+ 'Unknown': {
+   'title': 'Unknown',
+   'index': '6',
+   'level': 'info',
+
+   'description': 'Unknown is either a security problem that has not been assigned ' +
+                  'to a priority yet or a priority that our system did not recognize',
+   'banner_required': False
+ },
+
+ 'Negligible': {
+   'title': 'Negligible',
+   'index': '5',
+   'level': 'info',
+
+   'description': 'Negligible is technically a security problem, but is only theoretical ' +
+                  'in nature, requires a very special situation, has almost no install base, ' +
+                  'or does no real damage.',
+   'banner_required': False
+ },
+
+ 'Low': {
+   'title': 'Low',
+   'index': '4',
+   'level': 'warning',
+
+   'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
+                  'requires a user-assisted attack, a small install base, or does very ' +
+                  'little damage.',
+   'banner_required': False
+ },
+
+ 'Medium': {
+   'title': 'Medium',
+   'value': 'Medium',
+   'index': '3',
+   'level': 'warning',
+
+   'description': 'Medium is a real security problem, and is exploitable for many people. ' +
+                  'Includes network daemon denial of service attacks, cross-site scripting, ' +
+                  'and gaining user privileges.',
+   'banner_required': False
+ },
+
+ 'High': {
+   'title': 'High',
+   'value': 'High',
+   'index': '2',
+   'level': 'warning',
+
+   'description': 'High is a real problem, exploitable for many people in a default installation. ' +
+                  'Includes serious remote denial of services, local root privilege escalations, ' +
+                  'or data loss.',
+   'banner_required': False
+ },
+
+ 'Critical': {
+   'title': 'Critical',
+   'value': 'Critical',
+   'index': '1',
+   'level': 'error',
+
+   'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
+                  'a installation of the package. Includes remote root privilege escalations, ' +
+                  'or massive data loss.',
+   'banner_required': True
+ },
+
+ 'Defcon1': {
+   'title': 'Defcon 1',
+   'value': 'Defcon1',
+   'index': '0',
+   'level': 'error',
+
+   'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +
+                  'by the Quay team. It requires immediate attention.',
+   'banner_required': True
+ }
+}
+
+
+def get_priority_for_index(index):
+ for priority in PRIORITY_LEVELS:
+   if PRIORITY_LEVELS[priority]['index'] == index:
+     return priority
+
+ return 'Unknown'
+
+class SecurityConfigValidator(object):
+  def __init__(self, app, config_provider):
+    self._config_provider = config_provider
+
+    if not features.SECURITY_SCANNER:
+      return
+
+    self._security_config = app.config['SECURITY_SCANNER']
+    if self._security_config is None:
+      return
+
+    self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False
+    self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME')
+    self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME')
+
+    if self._public_key and self._private_key:
+      self._keys = (self._public_key, self._private_key)
+    else:
+      self._keys = None
+
+  def _get_filepath(self, key):
+    config = self._security_config
+
+    if key in config:
+      with self._config_provider.get_volume_file(config[key]) as f:
+        return f.name
+
+    return None
+
+  def cert(self):
+    return self._certificate
+
+  def keypair(self):
+    return self._keys
+
+  def valid(self):
+    if not features.SECURITY_SCANNER:
+      return False
+
+    if not self._security_config:
+      logger.debug('Missing SECURITY_SCANNER block in configuration')
+      return False
+
+    if not 'ENDPOINT' in self._security_config:
+      logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration')
+      return False
+
+    endpoint = self._security_config['ENDPOINT'] or ''
+    if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
+      logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https')
+      return False
+
+    if endpoint.startswith('https://') and (self._certificate is False or self._keys is None):
+      logger.debug('Certificate and key pair required for talking to security worker over HTTPS')
+      return False
+
+    return True
+
+
+class SecurityScannerAPI(object):
+  """ Helper class for talking to the Security Scan service (Clair). """
+  def __init__(self, app, config_provider):
+    self.app = app
+    self.config_provider = config_provider
+    self._security_config = None
+
+    config_validator = SecurityConfigValidator(app, config_provider)
+    if not config_validator.valid():
+      logger.warning('Invalid config provided to SecurityScannerAPI')
+      return
+
+    self._security_config = app.config.get('SECURITY_SCANNER')
+    self._certificate = config_validator.cert()
+    self._keys = config_validator.keypair()
+
+  def check_layer_vulnerable(self, layer_id, cve_id):
+    """ Checks with Clair whether the given layer is vulnerable to the given CVE. """
+    try:
+      body = {
+        'LayersIDs': [layer_id]
+      }
+      response = self.call('vulnerabilities/%s/affected-layers', body, cve_id)
+    except requests.exceptions.RequestException:
+      logger.exception('Got exception when trying to call Clair endpoint')
+      return False
+
+    if response.status_code != 200:
+      return False
+
+    try:
+      response_data = response.json()
+    except ValueError:
+      logger.exception('Got exception when trying to parse Clair response')
+      return False
+
+    if (not layer_id in response_data or
+        not response_data[layer_id].get('Vulnerable', False)):
+      return False
+
+    return True
+
+  def call(self, relative_url, body=None, *args, **kwargs):
+    """ Issues an HTTP call to the sec API at the given relative URL.
+        This function disconnects from the database while awaiting a response
+        from the API server.
+    """
+    security_config = self._security_config
+    if security_config is None:
+      raise Exception('Cannot call unconfigured security system')
+
+    api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
+    url = urljoin(api_url, relative_url % args)
+
+    client = self.app.config['HTTPCLIENT']
+    timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
+    logger.debug('Looking up sec information: %s', url)
+
+    with CloseForLongOperation(self.app.config):
+      if body is not None:
+        return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self._keys,
+                           verify=self._certificate)
+      else:
+        return client.get(url, params=kwargs, timeout=timeout, cert=self._keys,
+                          verify=self._certificate)

util/secscan/secscanendpoint.py

@@ -1,50 +0,0 @@
-import features
-import logging
-import requests
-import json
-
-from urlparse import urljoin
-
-logger = logging.getLogger(__name__)
-
-class SecurityScanEndpoint(object):
-  """ Helper class for talking to the Security Scan service (Clair). """
-  def __init__(self, app, config_provider):
-    self.app = app
-    self.config_provider = config_provider
-
-    if not features.SECURITY_SCANNER:
-      return
-
-    self.security_config = app.config['SECURITY_SCANNER']
-
-    self.certificate = self._getfilepath('CA_CERTIFICATE_FILENAME') or False
-    self.public_key = self._getfilepath('PUBLIC_KEY_FILENAME')
-    self.private_key = self._getfilepath('PRIVATE_KEY_FILENAME')
-
-    if self.public_key and self.private_key:
-      self.keys = (self.public_key, self.private_key)
-    else:
-      self.keys = None
-
-  def _getfilepath(self, config_key):
-    security_config = self.security_config
-
-    if config_key in security_config:
-      with self.config_provider.get_volume_file(security_config[config_key]) as f:
-        return f.name
-
-    return None
-
-  def call_api(self, relative_url, *args, **kwargs):
-    """ Issues an HTTP call to the sec API at the given relative URL. """
-    security_config = self.security_config
-    api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
-    url = urljoin(api_url, relative_url % args)
-
-    client = self.app.config['HTTPCLIENT']
-    timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
-    logger.debug('Looking up sec information: %s', url)
-
-    return client.get(url, params=kwargs, timeout=timeout, cert=self.keys,
-                      verify=self.certificate)

web.py

@@ -11,6 +11,7 @@ from endpoints.oauthlogin import oauthlogin
 from endpoints.githubtrigger import githubtrigger
 from endpoints.gitlabtrigger import gitlabtrigger
 from endpoints.bitbuckettrigger import bitbuckettrigger
+from endpoints.secscan import secscan
 
 if os.environ.get('DEBUGLOG') == 'true':
   logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
@@ -23,3 +24,4 @@ application.register_blueprint(bitbuckettrigger, url_prefix='/oauth1')
 application.register_blueprint(api_bp, url_prefix='/api')
 application.register_blueprint(webhooks, url_prefix='/webhooks')
 application.register_blueprint(realtime, url_prefix='/realtime')
+application.register_blueprint(secscan, url_prefix='/secscan')

workers/notificationworker.py

@@ -34,6 +34,7 @@ class NotificationWorker(QueueWorker):
       logger.exception('Cannot find notification event: %s', ex.message)
       raise JobException('Cannot find notification event: %s' % ex.message)
 
+    if event_handler.should_perform(job_details['event_data'], notification):
       method_handler.perform(notification, event_handler, job_details)
 
 

workers/security_notification_worker.py

@@ -0,0 +1,91 @@
+import json
+import logging
+import time
+
+from collections import defaultdict
+
+import features
+
+from app import secscan_notification_queue, secscan_api
+from data import model
+from data.database import (Image, ImageStorage, ExternalNotificationEvent,
+                           Repository, RepositoryNotification, RepositoryTag)
+from endpoints.notificationhelper import spawn_notification
+from workers.queueworker import QueueWorker
+
+
+logger = logging.getLogger(__name__)
+
+
+class SecurityNotificationWorker(QueueWorker):
+  def process_queue_item(self, data):
+    cve_id = data['Name']
+    vulnerability = data['Content']['Vulnerability']
+    priority = vulnerability['Priority']
+
+    # Lookup the external event for when we have vulnerabilities.
+    event = ExternalNotificationEvent.get(name='vulnerability_found')
+
+    # For each layer, retrieving the matching tags and join with repository to determine which
+    # require new notifications.
+    tag_map = defaultdict(set)
+    repository_map = {}
+
+    # Find all tags that contain the layer(s) introducing the vulnerability.
+    content = data['Content']
+    layer_ids = content.get('NewIntroducingLayersIDs', content.get('IntroducingLayersIDs', []))
+    for layer_id in layer_ids:
+      (docker_image_id, storage_uuid) = layer_id.split('.', 2)
+      tags = model.tag.get_matching_tags(docker_image_id, storage_uuid, RepositoryTag,
+                                         Repository, Image, ImageStorage)
+
+      # Additionally filter to tags only in repositories that have the event setup.
+      matching = list(tags
+                      .switch(RepositoryTag)
+                      .join(Repository)
+                      .join(RepositoryNotification)
+                      .where(RepositoryNotification.event == event))
+
+      check_map = {}
+      for tag in matching:
+        # Verify that the tag's root image has the vulnerability.
+        tag_layer_id = '%s.%s' % (tag.image.docker_image_id, tag.image.storage.uuid)
+        logger.debug('Checking if layer %s is vulnerable to %s', tag_layer_id, cve_id)
+
+        if not tag_layer_id in check_map:
+          is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id)
+          check_map[tag_layer_id] = is_vulerable
+
+        logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id,
+                     check_map[tag_layer_id])
+
+        if check_map[tag_layer_id]:
+          # Add the vulnerable tag to the list.
+          tag_map[tag.repository_id].add(tag.name)
+          repository_map[tag.repository_id] = tag.repository
+
+    # For each of the tags found, issue a notification.
+    for repository_id in tag_map:
+      tags = tag_map[repository_id]
+      event_data = {
+        'tags': list(tags),
+        'vulnerability': {
+          'id': data['Name'],
+          'description': vulnerability['Description'],
+          'link': vulnerability['Link'],
+          'priority': priority,
+        },
+      }
+
+      spawn_notification(repository_map[repository_id], 'vulnerability_found', event_data)
+
+
+if __name__ == '__main__':
+  if not features.SECURITY_SCANNER:
+    logger.debug('Security scanner disabled; skipping SecurityNotificationWorker')
+    while True:
+      time.sleep(100000)
+
+  worker = SecurityNotificationWorker(secscan_notification_queue, poll_period_seconds=30,
+                                      reservation_seconds=30, retry_after_seconds=30)
+  worker.start()

workers/securityworker.py

@@ -1,70 +1,96 @@
 import logging
+import logging.config
+
 import requests
 import features
 import time
 import os
 import random
 
-from sys import exc_info
+from endpoints.notificationhelper import spawn_notification
+from collections import defaultdict
 from peewee import JOIN_LEFT_OUTER
-from app import app, storage, OVERRIDE_CONFIG_DIRECTORY
+from app import app, config_provider, storage, OVERRIDE_CONFIG_DIRECTORY, secscan_api
 from workers.worker import Worker
-from data.database import Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement, db_random_func, UseThenDisconnect
+from data.database import (Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement,
+                           db_random_func, UseThenDisconnect, RepositoryTag, Repository,
+                           ExternalNotificationEvent, RepositoryNotification)
+from util.secscan.api import SecurityConfigValidator
 
 logger = logging.getLogger(__name__)
 
 BATCH_SIZE = 20
 INDEXING_INTERVAL = 10
-API_METHOD_INSERT = '/layers'
+API_METHOD_INSERT = '/v1/layers'
-API_METHOD_VERSION = '/versions/engine'
+API_METHOD_VERSION = '/v1/versions/engine'
 
-def _get_image_to_export(version):
+def _get_images_to_export_list(version):
   Parent = Image.alias()
   ParentImageStorage = ImageStorage.alias()
   rimages = []
 
-  # Without parent
+  # Collect the images without parents
   candidates = (Image
-    .select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum)
+                .select(Image.id, Image.docker_image_id, ImageStorage.uuid)
                 .join(ImageStorage)
-    .where(Image.security_indexed_engine < version, Image.parent_id >> None, ImageStorage.uploading == False, ImageStorage.checksum != '')
+                .where(Image.security_indexed_engine < version,
+                       Image.parent_id >> None,
+                       ImageStorage.uploading == False)
                 .limit(BATCH_SIZE*10)
                 .alias('candidates'))
 
   images = (Image
-    .select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum)
+            .select(candidates.c.id, candidates.c.docker_image_id, candidates.c.uuid)
             .from_(candidates)
             .order_by(db_random_func())
             .tuples()
             .limit(BATCH_SIZE))
 
   for image in images:
-    rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
+    rimages.append({'image_id': image[0],
+                    'docker_image_id': image[1],
+                    'storage_uuid': image[2],
+                    'parent_docker_image_id': None,
+                    'parent_storage_uuid': None})
 
-  # With analyzed parent
+  # Collect the images with analyzed parents.
   candidates = (Image
-      .select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum, Parent.docker_image_id.alias('parent_docker_image_id'), ParentImageStorage.uuid.alias('parent_storage_uuid'))
+                .select(Image.id,
+                        Image.docker_image_id,
+                        ImageStorage.uuid,
+                        Parent.docker_image_id.alias('parent_docker_image_id'),
+                        ParentImageStorage.uuid.alias('parent_storage_uuid'))
                 .join(Parent, on=(Image.parent_id == Parent.id))
                 .join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage))
                 .switch(Image)
                 .join(ImageStorage)
-      .where(Image.security_indexed_engine < version, Parent.security_indexed == True, Parent.security_indexed_engine >= version, ImageStorage.uploading == False, ImageStorage.checksum != '')
+                .where(Image.security_indexed_engine < version,
+                       Parent.security_indexed == True,
+                       Parent.security_indexed_engine >= version,
+                       ImageStorage.uploading == False)
                 .limit(BATCH_SIZE*10)
                 .alias('candidates'))
 
   images = (Image
-    .select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum, candidates.c.parent_docker_image_id, candidates.c.parent_storage_uuid)
+            .select(candidates.c.id,
+                    candidates.c.docker_image_id,
+                    candidates.c.uuid,
+                    candidates.c.parent_docker_image_id,
+                    candidates.c.parent_storage_uuid)
             .from_(candidates)
             .order_by(db_random_func())
             .tuples()
             .limit(BATCH_SIZE))
 
   for image in images:
-    rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': image[3], 'parent_storage_uuid': image[4]})
+    rimages.append({'image_id': image[0],
+                    'docker_image_id': image[1],
+                    'storage_uuid': image[2],
+                    'parent_docker_image_id': image[3],
+                    'parent_storage_uuid': image[4]})
 
-  # Re-shuffle, otherwise the images without parents will always be on the top
+  # Shuffle the images, otherwise the images without parents will always be on the top
   random.shuffle(rimages)
-
   return rimages
 
 def _get_storage_locations(uuid):
@@ -75,143 +101,212 @@ def _get_storage_locations(uuid):
            .join(ImageStorage, JOIN_LEFT_OUTER)
            .where(ImageStorage.uuid == uuid))
 
-  locations = list()
+  return [location.location.name for location in query]
-  for location in query:
-    locations.append(location.location.name)
-
-  return locations
 
 def _update_image(image, indexed, version):
   query = (Image
            .select()
            .join(ImageStorage)
-      .where(Image.docker_image_id == image['docker_image_id'], ImageStorage.uuid == image['storage_uuid']))
+           .where(Image.docker_image_id == image['docker_image_id'],
+                  ImageStorage.uuid == image['storage_uuid']))
 
-  updated_images = list()
+  ids_to_update = [row.id for row in query]
-  for image in query:
+  if not ids_to_update:
-    updated_images.append(image.id)
+    return
 
-  query = (Image
+  (Image
    .update(security_indexed=indexed, security_indexed_engine=version)
-      .where(Image.id << updated_images))
+   .where(Image.id << ids_to_update)
-  query.execute()
+   .execute())
+
 
 class SecurityWorker(Worker):
   def __init__(self):
     super(SecurityWorker, self).__init__()
-    if self._load_configuration():
+    validator = SecurityConfigValidator(app, config_provider)
-      self.add_operation(self._index_images, INDEXING_INTERVAL)
+    if validator.valid():
-
+      secscan_config = app.config.get('SECURITY_SCANNER')
-  def _load_configuration(self):
+      self._api = secscan_config['ENDPOINT']
-    # Load configuration
+      self._target_version = secscan_config['ENGINE_VERSION_TARGET']
-    config = app.config.get('SECURITY_SCANNER')
-
-    if not config or not 'ENDPOINT' in config or not 'ENGINE_VERSION_TARGET' in config or not 'DISTRIBUTED_STORAGE_PREFERENCE' in app.config:
-      logger.exception('No configuration found for the security worker')
-      return False
-    self._api = config['ENDPOINT']
-    self._target_version = config['ENGINE_VERSION_TARGET']
       self._default_storage_locations = app.config['DISTRIBUTED_STORAGE_PREFERENCE']
+      self._cert = validator.cert()
+      self._keys = validator.keypair()
 
-    self._ca_verification = False
+      self.add_operation(self._index_images, INDEXING_INTERVAL)
-    self._cert = None
+    logger.warning('Failed to validate security scan configuration')
-    if 'CA_CERTIFICATE_FILENAME' in config:
-      self._ca_verification = os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['CA_CERTIFICATE_FILENAME'])
-      if not os.path.isfile(self._ca_verification):
-        logger.exception('Could not find configured CA file')
-        return False
-    if 'PRIVATE_KEY_FILENAME' in config and 'PUBLIC_KEY_FILENAME' in config:
-      self._cert = (
-        os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['PUBLIC_KEY_FILENAME']),
-        os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['PRIVATE_KEY_FILENAME']),
-      )
-      if not os.path.isfile(self._cert[0]) or not os.path.isfile(self._cert[1]):
-        logger.exception('Could not find configured key pair files')
-        return False
 
-    return True
+  def _get_image_url(self, image):
-
+    """ Gets the download URL for an image and if the storage doesn't exist,
-  def _index_images(self):
+        marks the image as unindexed. """
-    with UseThenDisconnect(app.config):
+    path = storage.image_layer_path(image['storage_uuid'])
-      while True:
-        # Get images to analyze
-        try:
-          images = _get_image_to_export(self._target_version)
-        except Image.DoesNotExist:
-          logger.debug('No more image to analyze')
-          return
-
-        for img in images:
-          # Get layer storage URL
-          path = storage.image_layer_path(img['storage_uuid'])
     locations = self._default_storage_locations
-          if not storage.exists(locations, path):
-            locations = _get_storage_locations(img['storage_uuid'])
-          if not storage.exists(locations, path):
-            logger.warning('Could not find a valid location to download layer %s', img['docker_image_id']+'.'+img['storage_uuid'])
-            # Mark as analyzed because that error is most likely to occur during the pre-process, with the database copy
-            # when images are actually removed on the real database (and therefore in S3)
-            _update_image(img, False, self._target_version)
-            continue
-          uri = storage.get_direct_download_url(locations, path)
-          if uri == None:
-            # Local storage hack
-            uri = path
 
-          # Forge request
+    if not storage.exists(locations, path):
+      locations = _get_storage_locations(image['storage_uuid'])
+
+      if not locations or not storage.exists(locations, path):
+        logger.warning('Could not find a valid location to download layer %s',
+                       image['docker_image_id']+'.'+image['storage_uuid'])
+        _update_image(image, False, self._target_version)
+        return None
+
+    uri = storage.get_direct_download_url(locations, path)
+    if uri is None:
+      # Handle local storage
+      local_storage_enabled = False
+      for storage_type, _ in app.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
+        if storage_type == 'LocalStorage':
+          local_storage_enabled = True
+
+      if local_storage_enabled:
+        uri = path
+      else:
+        logger.warning('Could not get image URL and local storage was not enabled')
+        return None
+
+    return uri
+
+  def _new_request(self, image):
+    url = self._get_image_url(image)
+    if url is None:
+      return None
+
     request = {
-            'ID': img['docker_image_id']+'.'+img['storage_uuid'],
+      'ID': '%s.%s' % (image['docker_image_id'], image['storage_uuid']),
-            'TarSum': img['storage_checksum'],
+      'Path': url,
-            'Path': uri
     }
 
-          if img['parent_docker_image_id'] is not None and img['parent_storage_uuid'] is not None:
+    if image['parent_docker_image_id'] is not None and image['parent_storage_uuid'] is not None:
-            request['ParentID'] = img['parent_docker_image_id']+'.'+img['parent_storage_uuid']
+      request['ParentID'] = '%s.%s' % (image['parent_docker_image_id'],
+                                       image['parent_storage_uuid'])
 
-          # Post request
+    return request
+
+  def _analyze_image(self, image):
+    """ Analyzes an image by passing it to Clair. Returns the vulnerabilities detected
+        (if any) or None on error.
+    """
+    request = self._new_request(image)
+    if request is None:
+      return None
+
+    # Analyze the image.
     try:
       logger.info('Analyzing %s', request['ID'])
       # Using invalid certificates doesn't return proper errors because of
       # https://github.com/shazow/urllib3/issues/556
-            httpResponse = requests.post(self._api + API_METHOD_INSERT, json=request, cert=self._cert, verify=self._ca_verification)
+      httpResponse = requests.post(self._api + API_METHOD_INSERT, json=request,
-          except:
+                                   cert=self._keys, verify=self._cert)
-            logger.exception('An exception occurred when analyzing layer ID %s : %s', request['ID'], exc_info()[0])
-            return
-          try:
       jsonResponse = httpResponse.json()
-          except:
+    except (requests.exceptions.RequestException, ValueError):
-            logger.exception('An exception occurred when analyzing layer ID %s : the response is not valid JSON (%s)', request['ID'], httpResponse.text)
+      logger.exception('An exception occurred when analyzing layer ID %s', request['ID'])
-            return
+      return None
 
-          if httpResponse.status_code == 201:
+    # Handle any errors from the security scanner.
-            # The layer has been successfully indexed
+    if httpResponse.status_code != 201:
+      if 'OS and/or package manager are not supported' in jsonResponse.get('Message', ''):
+        # The current engine could not index this layer
+        logger.warning('A warning event occurred when analyzing layer ID %s : %s',
+                       request['ID'], jsonResponse['Message'])
+
+        # Hopefully, there is no version lower than the target one running
+        _update_image(image, False, self._target_version)
+      else:
+        logger.warning('Got non-201 when analyzing layer ID %s: %s', request['ID'], jsonResponse)
+
+      return None
+
+    # Verify that the version matches.
     api_version = jsonResponse['Version']
     if api_version < self._target_version:
       logger.warning('An engine runs on version %d but the target version is %d')
-            _update_image(img, True, api_version)
+
-            logger.info('Layer ID %s : analyzed successfully', request['ID'])
+    # Mark the image as analyzed.
-          else:
+    logger.debug('Layer %s analyzed successfully; Loading vulnerabilities for layer',
-            if 'Message' in jsonResponse:
+                 image['image_id'])
-              if 'OS and/or package manager are not supported' in jsonResponse['Message']:
+    _update_image(image, True, api_version)
-                # The current engine could not index this layer
+
-                logger.warning('A warning event occurred when analyzing layer ID %s : %s', request['ID'], jsonResponse['Message'])
+    # Lookup the vulnerabilities for the image, now that it is analyzed.
-                # Hopefully, there is no version lower than the target one running
+    try:
-                _update_image(img, False, self._target_version)
+      response = secscan_api.call('layers/%s/vulnerabilities', None, request['ID'])
-              else:
+      logger.debug('Got response %s for vulnerabilities for layer %s',
-                logger.exception('An exception occurred when analyzing layer ID %s : %d %s', request['ID'], httpResponse.status_code, jsonResponse['Message'])
+                   response.status_code, image['image_id'])
-                return
+      if response.status_code == 404:
-            else:
+        return None
-              logger.exception('An exception occurred when analyzing layer ID %s : %d', request['ID'], httpResponse.status_code)
+    except (requests.exceptions.RequestException, ValueError):
+      logger.exception('Failed to get vulnerability response for %s', image['image_id'])
+      return None
+
+    return response.json()
+
+  def _index_images(self):
+    logger.debug('Started indexing')
+
+    with UseThenDisconnect(app.config):
+      while True:
+        # Lookup the images to index.
+        images = []
+        try:
+          logger.debug('Looking up images to index')
+          images = _get_images_to_export_list(self._target_version)
+        except Image.DoesNotExist:
+          pass
+
+        if not images:
+          logger.debug('No more images left to analyze')
           return
 
+        logger.debug('Found %d images to index', len(images))
+        for image in images:
+          # Analyze the image, retrieving the vulnerabilities (if any).
+          sec_data = self._analyze_image(image)
+          if sec_data is None:
+            continue
+
+          if not sec_data.get('Vulnerabilities'):
+            continue
+
+          # Dispatch events for any detected vulnerabilities
+          logger.debug('Got vulnerabilities for layer %s: %s', image['image_id'], sec_data)
+          event = ExternalNotificationEvent.get(name='vulnerability_found')
+          matching = (RepositoryTag
+                      .select(RepositoryTag, Repository)
+                      .distinct()
+                      .join(Repository)
+                      .join(RepositoryNotification)
+                      .where(RepositoryNotification.event == event,
+                             RepositoryTag.image == image['image_id'],
+                             RepositoryTag.hidden == False,
+                             RepositoryTag.lifetime_end_ts >> None))
+
+          repository_map = defaultdict(list)
+
+          for tag in matching:
+            repository_map[tag.repository_id].append(tag)
+
+          for repository_id in repository_map:
+            tags = repository_map[repository_id]
+
+            for vuln in sec_data['Vulnerabilities']:
+              event_data = {
+                'tags': [tag.name for tag in tags],
+                'vulnerability': {
+                  'id': vuln['ID'],
+                  'description': vuln['Description'],
+                  'link': vuln['Link'],
+                  'priority': vuln['Priority'],
+                },
+              }
+
+              spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
+
+
 if __name__ == '__main__':
-  logging.getLogger('requests').setLevel(logging.WARNING)
-  logging.getLogger('apscheduler').setLevel(logging.CRITICAL)
-
   if not features.SECURITY_SCANNER:
-    logger.debug('Security scanner disabled; skipping')
+    logger.debug('Security scanner disabled; skipping SecurityWorker')
     while True:
       time.sleep(100000)
 
+  logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
   worker = SecurityWorker()
   worker.start()

workers/worker.py

@@ -61,7 +61,7 @@ class Worker(object):
       pass
 
   def start(self):
-    logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
+    logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
 
     if not app.config.get('SETUP_COMPLETE', False):
       logger.info('Product setup is not yet complete; skipping worker startup')