Merge pull request #820 from coreos-inc/phase1-11-07-2015

Phase1 11 07 2015

app.py

@@ -35,6 +35,7 @@ from util.saas.metricqueue import MetricQueue
 from util.config.provider import get_config_provider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
+from util.secscan.secscanendpoint import SecurityScanEndpoint
 
 OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
 OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
@@ -147,6 +148,7 @@ image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf)
 dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
                                    reporter=MetricQueueReporter(metric_queue))
 notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
+secscan_endpoint = SecurityScanEndpoint(app, config_provider)
 
 database.configure(app.config)
 model.config.app_config = app.config

conf/init/service/securityworker/log/run

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec logger -i -t securityworker

conf/init/service/securityworker/run

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+echo 'Starting security scanner worker'
+
+cd /
+venv/bin/python -m workers.securityworker 2>&1
+
+echo 'Security scanner worker exited'

config.py

@@ -250,3 +250,12 @@ class DefaultConfig(object):
 
   # Experiment: Async garbage collection
   EXP_ASYNC_GARBAGE_COLLECTION = []
+
+  # Security scanner
+  FEATURE_SECURITY_SCANNER = False
+  SECURITY_SCANNER = {
+    'ENDPOINT': 'http://192.168.99.100:6060',
+    'ENGINE_VERSION_TARGET': 1,
+    'API_VERSION': 'v1',
+    'API_TIMEOUT_SECONDS': 10,
+  }

data/database.py

@@ -472,9 +472,6 @@ class RepositoryBuildTrigger(BaseModel):
   pull_robot = QuayUserField(allows_robots=True, null=True, related_name='triggerpullrobot',
                              robot_null_delete=True)
 
-  # TODO(jschorr): Remove this column once we verify the backfill has succeeded.
-  used_legacy_github = BooleanField(null=True, default=False)
-
 
 class EmailConfirmation(BaseModel):
   code = CharField(default=random_string_generator(), unique=True, index=True)
@@ -487,11 +484,12 @@ class EmailConfirmation(BaseModel):
 
 class ImageStorage(BaseModel):
   uuid = CharField(default=uuid_generator, index=True, unique=True)
-  checksum = CharField(null=True)
+  checksum = CharField(null=True)  # TODO remove when all checksums have been moved back to Image
   image_size = BigIntegerField(null=True)
   uncompressed_size = BigIntegerField(null=True)
   uploading = BooleanField(default=True, null=True)
   cas_path = BooleanField(default=True)
+  content_checksum = CharField(null=True, index=True)
 
 
 class ImageStorageTransformation(BaseModel):
@@ -573,6 +571,11 @@ class Image(BaseModel):
   command = TextField(null=True)
   aggregate_size = BigIntegerField(null=True)
   v1_json_metadata = TextField(null=True)
+  v1_checksum = CharField(null=True)
+
+  security_indexed = BooleanField(default=False)
+  security_indexed_engine = IntegerField(default=-1)
+  parent = ForeignKeyField('self', index=True, null=True, related_name='children')
 
   class Meta:
     database = db
@@ -580,6 +583,8 @@ class Image(BaseModel):
     indexes = (
       # we don't really want duplicates
       (('repository', 'docker_image_id'), True),
+
+      (('security_indexed_engine', 'security_indexed'), False),
     )
 
 
@@ -746,6 +751,7 @@ class RepositoryNotification(BaseModel):
   method = ForeignKeyField(ExternalNotificationMethod)
   title = CharField(null=True)
   config_json = TextField()
+  event_config_json = TextField(default='{}')
 
 
 class RepositoryAuthorizedEmail(BaseModel):

data/migrations/versions/2827d36939e4_separate_v1_and_v2_checksums.py

@@ -0,0 +1,30 @@
+"""Separate v1 and v2 checksums.
+
+Revision ID: 2827d36939e4
+Revises: 73669db7e12
+Create Date: 2015-11-04 16:29:48.905775
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '2827d36939e4'
+down_revision = '5cdc2d819c5'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('image', sa.Column('v1_checksum', sa.String(length=255), nullable=True))
+    op.add_column('imagestorage', sa.Column('content_checksum', sa.String(length=255), nullable=True))
+    op.create_index('imagestorage_content_checksum', 'imagestorage', ['content_checksum'], unique=False)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('imagestorage_content_checksum', table_name='imagestorage')
+    op.drop_column('imagestorage', 'content_checksum')
+    op.drop_column('image', 'v1_checksum')
+    ### end Alembic commands ###

data/migrations/versions/50925110da8c_add_event_specific_config.py

@@ -0,0 +1,27 @@
+"""Add event-specific config
+
+Revision ID: 50925110da8c
+Revises: 2fb9492c20cc
+Create Date: 2015-10-13 18:03:14.859839
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '50925110da8c'
+down_revision = '57dad559ff2d'
+
+from alembic import op
+import sqlalchemy as sa
+from util.migrate import UTF8LongText
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorynotification', sa.Column('event_config_json', UTF8LongText, nullable=False))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorynotification', 'event_config_json')
+    ### end Alembic commands ###

data/migrations/versions/57dad559ff2d_add_support_for_quay_s_security_indexer.py

@@ -0,0 +1,32 @@
+"""add support for quay's security indexer
+Revision ID: 57dad559ff2d
+Revises: 154f2befdfbe
+Create Date: 2015-07-13 16:51:41.669249
+"""
+
+# revision identifiers, used by Alembic.
+revision = '57dad559ff2d'
+down_revision = '73669db7e12'
+
+from alembic import op
+import sqlalchemy as sa
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('image', sa.Column('parent_id', sa.Integer(), nullable=True))
+    op.add_column('image', sa.Column('security_indexed', sa.Boolean(), nullable=False, default=False, server_default=sa.sql.expression.false()))
+    op.add_column('image', sa.Column('security_indexed_engine', sa.Integer(), nullable=False, default=-1, server_default="-1"))
+    op.create_index('image_parent_id', 'image', ['parent_id'], unique=False)
+    op.create_foreign_key(op.f('fk_image_parent_id_image'), 'image', 'image', ['parent_id'], ['id'])
+    ### end Alembic commands ###
+    op.create_index('image_security_indexed_engine_security_indexed', 'image', ['security_indexed_engine', 'security_indexed'])
+
+def downgrade(tables):
+   ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('image_security_indexed_engine_security_indexed', 'image')
+    op.drop_constraint(op.f('fk_image_parent_id_image'), 'image', type_='foreignkey')
+    op.drop_index('image_parent_id', table_name='image')
+    op.drop_column('image', 'security_indexed')
+    op.drop_column('image', 'security_indexed_engine')
+    op.drop_column('image', 'parent_id')
+    ### end Alembic commands ###

data/migrations/versions/5cdc2d819c5_add_vulnerability_found_event.py

@@ -0,0 +1,41 @@
+"""Add vulnerability_found event
+
+Revision ID: 5cdc2d819c5
+Revises: 50925110da8c
+Create Date: 2015-10-13 18:05:32.157858
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5cdc2d819c5'
+down_revision = '50925110da8c'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+
+def upgrade(tables):
+    op.bulk_insert(tables.externalnotificationevent,
+    [
+        {'id':6, 'name':'vulnerability_found'},
+    ])
+
+    op.bulk_insert(tables.notificationkind,
+    [
+        {'id':11, 'name':'vulnerability_found'},
+    ])
+
+
+def downgrade(tables):
+    op.execute(
+        (tables.externalnotificationevent.delete()
+            .where(tables.externalnotificationevent.c.name == op.inline_literal('vulnerability_found')))
+
+    )
+
+    op.execute(
+        (tables.notificationkind.delete()
+            .where(tables.notificationkind.c.name == op.inline_literal('vulnerability_found')))
+
+    )

data/migrations/versions/73669db7e12_remove_legacy_github_column.py

@@ -0,0 +1,25 @@
+"""Remove legacy github column
+
+Revision ID: 73669db7e12
+Revises: 35f538da62
+Create Date: 2015-11-04 16:18:18.107314
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '73669db7e12'
+down_revision = '35f538da62'
+
+from alembic import op
+import sqlalchemy as sa
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorybuildtrigger', 'used_legacy_github')
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorybuildtrigger', sa.Column('used_legacy_github', sa.Boolean(), nullable=True))
+    ### end Alembic commands ###

data/model/blob.py

@@ -17,7 +17,7 @@ def get_repo_blob_by_digest(namespace, repo_name, blob_digest):
                     .join(Repository)
                     .join(Namespace)
                     .where(Repository.name == repo_name, Namespace.username == namespace,
-                           ImageStorage.checksum == blob_digest))
+                           ImageStorage.content_checksum == blob_digest))
   if not placements:
     raise BlobDoesNotExist('Blob does not exist with digest: {0}'.format(blob_digest))
 
@@ -35,11 +35,11 @@ def store_blob_record_and_temp_link(namespace, repo_name, blob_digest, location_
     repo = _basequery.get_existing_repository(namespace, repo_name)
 
     try:
-      storage = ImageStorage.get(checksum=blob_digest)
+      storage = ImageStorage.get(content_checksum=blob_digest)
       location = ImageStorageLocation.get(name=location_name)
       ImageStoragePlacement.get(storage=storage, location=location)
     except ImageStorage.DoesNotExist:
-      storage = ImageStorage.create(checksum=blob_digest)
+      storage = ImageStorage.create(content_checksum=blob_digest)
     except ImageStoragePlacement.DoesNotExist:
       ImageStoragePlacement.create(storage=storage, location=location)
 

data/model/image.py

@@ -284,10 +284,7 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
     except Image.DoesNotExist:
       raise DataModelException('No image with specified id and repository')
 
-    # We cleanup any old checksum in case it's a retry after a fail
-    fetched.storage.checksum = None
     fetched.created = datetime.now()
-
     if created_date_str is not None:
       try:
         fetched.created = dateutil.parser.parse(created_date_str).replace(tzinfo=None)
@@ -295,12 +292,18 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
         # parse raises different exceptions, so we cannot use a specific kind of handler here.
         pass
 
+    # We cleanup any old checksum in case it's a retry after a fail
+    fetched.v1_checksum = None
+    fetched.storage.checksum = None # TODO remove when storage checksums are no longer read
+    fetched.storage.content_checksum = None
+
     fetched.comment = comment
     fetched.command = command
     fetched.v1_json_metadata = v1_json_metadata
 
     if parent:
       fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id)
+      fetched.parent = parent
 
     fetched.save()
     fetched.storage.save()

data/model/notification.py

@@ -113,12 +113,13 @@ def delete_matching_notifications(target, kind_name, **kwargs):
     notification.delete_instance()
 
 
-def create_repo_notification(repo, event_name, method_name, config, title=None):
+def create_repo_notification(repo, event_name, method_name, method_config, event_config, title=None):
   event = ExternalNotificationEvent.get(ExternalNotificationEvent.name == event_name)
   method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == method_name)
 
   return RepositoryNotification.create(repository=repo, event=event, method=method,
-                                       config_json=json.dumps(config), title=title)
+                                       config_json=json.dumps(method_config), title=title,
+                                       event_config_json=json.dumps(event_config))
 
 
 def get_repo_notification(uuid):

data/model/user.py

@@ -651,9 +651,12 @@ def detach_external_login(user, service_name):
 
 
 def delete_user(user):
-  user.delete_instance(recursive=True, delete_nullable=True)
+  # Delete any repositories under the user's namespace.
+  for repo in list(Repository.select().where(Repository.namespace_user == user)):
+    repository.purge_repository(user.username, repo.name)
 
-  # TODO: also delete any repository data associated
+  # Delete the user itself.
+  user.delete_instance(recursive=True, delete_nullable=True)
 
 
 def get_pull_credentials(robotname):

digest/checksums.py

@@ -75,6 +75,14 @@ def simple_checksum_handler(json_data):
     return h, fn
 
 
+def content_checksum_handler():
+    h = hashlib.sha256()
+
+    def fn(buf):
+        h.update(buf)
+    return h, fn
+
+
 def compute_simple(fp, json_data):
     data = json_data + '\n'
     return 'sha256:{0}'.format(sha256_file(fp, data))

endpoints/api/__init__.py

@@ -93,6 +93,11 @@ class NotFound(ApiException):
     ApiException.__init__(self, None, 404, 'Not Found', payload)
 
 
+class DownstreamIssue(ApiException):
+  def __init__(self, payload=None):
+    ApiException.__init__(self, None, 520, 'Downstream Issue', payload)
+
+
 @api_bp.app_errorhandler(ApiException)
 @crossdomain(origin='*', headers=['Authorization', 'Content-Type'])
 def handle_api_error(error):
@@ -418,4 +423,5 @@ import endpoints.api.tag
 import endpoints.api.team
 import endpoints.api.trigger
 import endpoints.api.user
+import endpoints.api.secscan
 

endpoints/api/repositorynotification.py

@@ -57,6 +57,10 @@ class RepositoryNotificationList(RepositoryParamResource):
           'type': 'object',
           'description': 'JSON config information for the specific method of notification'
         },
+        'eventConfig': {
+          'type': 'object',
+          'description':  'JSON config information for the specific event of notification',
+        },
         'title': {
           'type': 'string',
           'description': 'The human-readable title of the notification',
@@ -84,6 +88,7 @@ class RepositoryNotificationList(RepositoryParamResource):
 
     new_notification = model.notification.create_repo_notification(repo, parsed['event'],
                                                              parsed['method'], parsed['config'],
+                                                             parsed['eventConfig'],
                                                              parsed.get('title', None))
 
     resp = notification_view(new_notification)

endpoints/api/secscan.py

@@ -0,0 +1,103 @@
+""" List and manage repository vulnerabilities and other sec information. """
+
+import logging
+import features
+import json
+import requests
+
+from app import secscan_endpoint
+from data import model
+from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
+                           RepositoryParamResource, resource, nickname, show_if, parse_args,
+                           query_param)
+
+
+logger = logging.getLogger(__name__)
+
+
+def _call_security_api(relative_url, *args, **kwargs):
+  """ Issues an HTTP call to the sec API at the given relative URL. """
+  try:
+    response = secscan_endpoint.call_api(relative_url, *args, **kwargs)
+  except requests.exceptions.Timeout:
+    raise DownstreamIssue(payload=dict(message='API call timed out'))
+  except requests.exceptions.ConnectionError:
+    raise DownstreamIssue(payload=dict(message='Could not connect to downstream service'))
+
+  if response.status_code == 404:
+    raise NotFound()
+
+  try:
+    response_data = json.loads(response.text)
+  except ValueError:
+    raise DownstreamIssue(payload=dict(message='Non-json response from downstream service'))
+
+  if response.status_code / 100 != 2:
+    logger.warning('Got %s status code to call: %s', response.status_code, response.text)
+    raise DownstreamIssue(payload=dict(message=response_data['Message']))
+
+  return response_data
+
+
+@show_if(features.SECURITY_SCANNER)
+@resource('/v1/repository/<repopath:repository>/tag/<tag>/vulnerabilities')
+@path_param('repository', 'The full path of the repository. e.g. namespace/name')
+@path_param('tag', 'The name of the tag')
+class RepositoryTagVulnerabilities(RepositoryParamResource):
+  """ Operations for managing the vulnerabilities in a repository tag. """
+
+  @require_repo_read
+  @nickname('getRepoTagVulnerabilities')
+  @parse_args
+  @query_param('minimumPriority', 'Minimum vulnerability priority', type=str,
+               default='Low')
+  def get(self, args, namespace, repository, tag):
+    """ Fetches the vulnerabilities (if any) for a repository tag. """
+    try:
+      tag_image = model.tag.get_tag_image(namespace, repository, tag)
+    except model.DataModelException:
+      raise NotFound()
+
+    if not tag_image.security_indexed:
+      logger.debug('Image %s for tag %s under repository %s/%s not security indexed',
+                   tag_image.docker_image_id, tag, namespace, repository)
+      return {
+        'security_indexed': False
+      }
+
+    data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id,
+                              minimumPriority=args.minimumPriority)
+
+    return {
+      'security_indexed': True,
+      'data': data,
+    }
+
+
+@show_if(features.SECURITY_SCANNER)
+@resource('/v1/repository/<repopath:repository>/image/<imageid>/packages')
+@path_param('repository', 'The full path of the repository. e.g. namespace/name')
+@path_param('imageid', 'The image ID')
+class RepositoryImagePackages(RepositoryParamResource):
+  """ Operations for listing the packages added/removed in an image. """
+
+  @require_repo_read
+  @nickname('getRepoImagePackages')
+  def get(self, namespace, repository, imageid):
+    """ Fetches the packages added/removed in the given repo image. """
+    repo_image = model.image.get_repo_image(namespace, repository, imageid)
+    if repo_image is None:
+      raise NotFound()
+
+    if not repo_image.security_indexed:
+      return {
+        'security_indexed': False
+      }
+
+    data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id)
+
+    return {
+      'security_indexed': True,
+      'data': data,
+    }
+

endpoints/notificationevent.py

@@ -84,6 +84,40 @@ def _build_summary(event_data):
   return summary
 
 
+class VulnerabilityFoundEvent(NotificationEvent):
+  @classmethod
+  def event_name(cls):
+    return 'vulnerability_found'
+
+  def get_level(self, event_data, notification_data):
+    priority = event_data['vulnerability']['priority']
+    if priority == 'Defcon1' or priority == 'Critical':
+      return 'error'
+
+    if priority == 'Medium' or priority == 'High':
+      return 'warning'
+
+    return 'info'
+
+  def get_sample_data(self, repository):
+    return build_event_data(repository, {
+      'tags': ['latest', 'prod'],
+      'image': 'some-image-id',
+      'vulnerability': {
+        'id': 'CVE-FAKE-CVE',
+        'description': 'A futurist vulnerability',
+        'link': 'https://security-tracker.debian.org/tracker/CVE-FAKE-CVE',
+        'priority': 'Critical',
+      },
+    })
+
+  def get_summary(self, event_data, notification_data):
+    msg = '%s vulnerability detected in repository %s in tags %s'
+    return msg % (event_data['vulnerability']['priority'],
+                  event_data['repository'],
+                  ', '.join(event_data['tags']))
+
+
 class BuildQueueEvent(NotificationEvent):
   @classmethod
   def event_name(cls):

endpoints/v1/registry.py

@@ -249,6 +249,10 @@ def put_image_layer(namespace, repository, image_id):
   h, sum_hndlr = checksums.simple_checksum_handler(json_data)
   sr.add_handler(sum_hndlr)
 
+  # Add a handler which computes the content checksum only
+  ch, content_sum_hndlr = checksums.content_checksum_handler()
+  sr.add_handler(content_sum_hndlr)
+
   # Stream write the data to storage.
   with database.CloseForLongOperation(app.config):
     try:
@@ -278,6 +282,7 @@ def put_image_layer(namespace, repository, image_id):
     # We don't have a checksum stored yet, that's fine skipping the check.
     # Not removing the mark though, image is not downloadable yet.
     session['checksum'] = csums
+    session['content_checksum'] = 'sha256:{0}'.format(ch.hexdigest())
     return make_response('true', 200)
 
   checksum = repo_image.storage.checksum
@@ -339,8 +344,9 @@ def put_image_checksum(namespace, repository, image_id):
     abort(409, 'Cannot set checksum for image %(image_id)s',
           issue='image-write-error', image_id=image_id)
 
-  logger.debug('Storing image checksum')
+  logger.debug('Storing image and content checksums')
-  err = store_checksum(repo_image.storage, checksum)
+  content_checksum = session.get('content_checksum', None)
+  err = store_checksum(repo_image, checksum, content_checksum)
   if err:
     abort(400, err)
 
@@ -429,14 +435,18 @@ def generate_ancestry(image_id, uuid, locations, parent_id=None, parent_uuid=Non
   store.put_content(locations, store.image_ancestry_path(uuid), json.dumps(data))
 
 
-def store_checksum(image_storage, checksum):
+def store_checksum(image_with_storage, checksum, content_checksum):
   checksum_parts = checksum.split(':')
   if len(checksum_parts) != 2:
     return 'Invalid checksum format'
 
   # We store the checksum
-  image_storage.checksum = checksum
+  image_with_storage.storage.checksum = checksum  # TODO remove when v1 checksums are on image only
-  image_storage.save()
+  image_with_storage.storage.content_checksum = content_checksum
+  image_with_storage.storage.save()
+
+  image_with_storage.v1_checksum = checksum
+  image_with_storage.save()
 
 
 @v1_bp.route('/images/<image_id>/json', methods=['PUT'])

events/vulnerability_found.html

@@ -0,0 +1,4 @@
+A <a href="{{ event_data.vulnerability.link }}">{{ event_data.vulnerability.priority }} vulnerability</a> ({{ event_data.vulnerability.id }}) was detected in tags
+ {{ 'tags' | icon_image }}
+{% for tag in event_data.tags %}{%if loop.index > 1 %}, {% endif %}{{ (event_data.repository, tag) | repository_tag_reference }}{% endfor %} in
+ repository {{ event_data.repository | repository_reference }}

initdb.py

@@ -5,6 +5,7 @@ import random
 import calendar
 import os
 
+from sys import maxsize
 from datetime import datetime, timedelta
 from peewee import (SqliteDatabase, create_model_tables, drop_model_tables, savepoint_sqlite,
                     savepoint)
@@ -82,7 +83,7 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
     new_image_locations = new_image.storage.locations
     new_image.storage.uuid = __gen_image_uuid(repo, image_num)
     new_image.storage.uploading = False
-    new_image.storage.checksum = checksum
+    new_image.storage.content_checksum = checksum
     new_image.storage.save()
 
     # Write some data for the storage.
@@ -95,6 +96,10 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
         path = path_builder(new_image.storage.uuid)
         store.put_content('local_us', path, checksum)
 
+    new_image.security_indexed = False
+    new_image.security_indexed_engine = maxsize
+    new_image.save()
+
     creation_time = REFERENCE_DATE + timedelta(weeks=image_num) + timedelta(days=model_num)
     command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
     command = json.dumps(command_list) if command_list else None
@@ -309,6 +314,7 @@ def initialize_database():
   ExternalNotificationEvent.create(name='build_start')
   ExternalNotificationEvent.create(name='build_success')
   ExternalNotificationEvent.create(name='build_failure')
+  ExternalNotificationEvent.create(name='vulnerability_found')
 
   ExternalNotificationMethod.create(name='quay_notification')
   ExternalNotificationMethod.create(name='email')
@@ -323,6 +329,7 @@ def initialize_database():
   NotificationKind.create(name='build_start')
   NotificationKind.create(name='build_success')
   NotificationKind.create(name='build_failure')
+  NotificationKind.create(name='vulnerability_found')
 
   NotificationKind.create(name='password_required')
   NotificationKind.create(name='over_private_usage')

static/css/directives/ui/create-external-notification-dialog.css

@@ -0,0 +1,20 @@
+#createNotificationModal .dropdown-select {
+  margin: 0px;
+}
+
+#createNotificationModal .options-table {
+  width: 100%;
+  margin-bottom: 10px;
+}
+
+#createNotificationModal .options-table td {
+  padding-bottom: 6px;
+}
+
+#createNotificationModal .options-table td.name {
+  width: 160px;
+}
+
+#createNotificationModal .options-table-wrapper {
+  padding: 10px;
+}

static/directives/create-external-notification-dialog.html

@@ -1,12 +1,12 @@
 <!-- Modal message dialog -->
-<div class="modal fade" id="createNotificationModal">
+<div class="co-dialog modal fade" id="createNotificationModal">
   <div class="modal-dialog">
     <div class="modal-content">
       <form id="createForm" name="createForm" ng-submit="createNotification()">
         <div class="modal-header">
           <button type="button" class="close" data-dismiss="modal" aria-hidden="true" ng-disabled="creating">&times;</button>
           <h4 class="modal-title">
-            Create Repository Notification
+            <i class="fa fa-bell"></i> Create Repository Notification
           </h4>
         </div>
         <div class="modal-body">
@@ -29,109 +29,127 @@
           </div>
 
           <!-- Create View -->
-          <table style="width: 100%" ng-show="status == ''">
+          <div class="options-table-wrapper">
-            <tr>
+            <table class="options-table" ng-show="status == ''">
-              <td style="width: 120px">Notification title:</td>
+              <tr>
-              <td style="padding-right: 21px;">
+                <td class="name">Notification title:</td>
-                <input class="form-control" type="text" placeholder="(Optional Title)" ng-model="currentTitle"
+                <td>
-                       style="margin: 10px;">
+                  <input class="form-control" type="text" placeholder="(Optional Title)" ng-model="currentTitle">
-              </td>
+                </td>
-            </tr>
+              </tr>
+            </table>
 
-            <tr>
+            <table class="options-table" ng-show="status == ''">
-              <td style="width: 120px">When this occurs:</td>
+              <tr>
-              <td>
+                <td class="name">When this occurs:</td>
-                <div class="dropdown-select" placeholder="'(Notification Event)'" selected-item="currentEvent.title"
+                <td>
-                     handle-item-selected="handleEventSelected(datum)" clear-value="clearCounter">
+                  <div class="dropdown-select" placeholder="'(Notification Event)'" selected-item="currentEvent.title"
-                  <!-- Icons -->
+                       handle-item-selected="handleEventSelected(datum)" clear-value="clearCounter">
-                  <i class="dropdown-select-icon fa fa-lg" ng-class="currentEvent.icon"></i>
+                    <!-- Icons -->
+                    <i class="dropdown-select-icon fa fa-lg" ng-class="currentEvent.icon"></i>
 
-                  <!-- Dropdown menu -->
+                    <!-- Dropdown menu -->
-                  <ul class="dropdown-select-menu pull-right" role="menu">
+                    <ul class="dropdown-select-menu pull-right" role="menu">
-                    <li ng-repeat="event in events">
+                      <li ng-repeat="event in events">
-                      <a href="javascript:void(0)" ng-click="setEvent(event)">
+                        <a href="javascript:void(0)" ng-click="setEvent(event)">
-                        <i class="fa fa-lg" ng-class="event.icon"></i> {{ event.title }}
+                          <i class="fa fa-lg" ng-class="event.icon"></i> {{ event.title }}
-                      </a>
+                        </a>
-                    </li>
+                      </li>
-                  </ul>
+                    </ul>
-                </div>
+                  </div>
-              </td>
+                </td>
-            </tr>
+              </tr>
 
-            <tr>
+              <tr ng-repeat="field in currentEvent.fields">
-              <td>Then issue a:</td>
+                <td class="name" valign="top">With {{ field.title }} of:</td>
-              <td>
+                <td>
-                <div class="dropdown-select" placeholder="'(Notification Action)'" selected-item="currentMethod.title"
+                  <div ng-switch on="field.type">
-                     handle-item-selected="handleMethodSelected(datum)" clear-value="clearCounter">
+                    <select class="form-control" ng-if="field.type == 'enum'"
-                  <!-- Icons -->
+                            ng-model="currentEventConfig[field.name]" required>
-                  <i class="dropdown-select-icon fa fa-lg" ng-class="currentMethod.icon"></i>
+                      <option ng-repeat="(key, info) in field.values | orderObjectBy: 'index'" value="{{key}}">{{ info.title }}</option>
+                    </select>
 
-                  <!-- Dropdown menu -->
+                    <div class="co-alert co-alert-info"
-                  <ul class="dropdown-select-menu pull-right" role="menu">
+                         style="margin-top: 6px; margin-bottom: 10px;"
-                    <li ng-repeat="method in methods">
+                         ng-if="field.values[currentEventConfig[field.name]].description">
-                      <a href="javascript:void(0)" ng-click="setMethod(method)">
+                      {{ field.values[currentEventConfig[field.name]].description }}
-                        <i class="fa fa-lg" ng-class="method.icon"></i> {{ method.title }}
-                      </a>
-                    </li>
-                  </ul>
-                </div>
-              </td>
-            </tr>
-
-            <tr ng-if="currentMethod.fields.length"><td colspan="2"><hr></td></tr>
-
-            <tr ng-repeat="field in currentMethod.fields">
-              <td valign="top" style="padding-top: 10px">{{ field.title }}:</td>
-              <td>
-                <div ng-switch on="field.type">
-                  <span ng-switch-when="email">
-                    <input type="email" class="form-control" ng-model="currentConfig[field.name]" required>
-                  </span>
-                  <input type="url" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="url" required>
-                  <input type="text" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="string" required>
-                  <!-- TODO(jschorr): unify the ability to create an input box with all the usual features -->
-                  <div ng-switch-when="regex">
-                    <input type="text" class="form-control" ng-model="currentConfig[field.name]"
-                           ng-pattern="getPattern(field)"
-                           placeholder="{{ field.placeholder }}"
-                           ng-name="field.name"
-                           id="{{ field.name }}"
-                           required>
-
-                    <div class="alert alert-warning" style="margin-top: 10px; margin-bottom: 10px"
-                          ng-if="field.regex_fail_message && hasRegexMismatch(createForm.$error, field.name)">
-                      <span ng-bind-html="field.regex_fail_message"></span>
                     </div>
                   </div>
-                  <div class="entity-search" namespace="repository.namespace"
+                </td>
-                       placeholder="''"
+              </tr>
-                       current-entity="currentConfig[field.name]"
+            </table>
-                       ng-model="currentConfig[field.name]"
-                       allowed-entities="['user', 'team', 'org']"
-                       ng-switch-when="entity"></div>
 
-                  <div ng-if="getHelpUrl(field, currentConfig)"
+            <table class="options-table" ng-show="status == ''">
-                       style="margin-top: 10px;  margin-bottom: 10px">
+              <tr>
-                    See: <a href="{{ getHelpUrl(field, currentConfig) }}" target="_blank">{{ getHelpUrl(field, currentConfig) }}</a>
+                <td class="name">Then issue a:</td>
+                <td>
+                  <div class="dropdown-select" placeholder="'(Notification Action)'" selected-item="currentMethod.title"
+                       handle-item-selected="handleMethodSelected(datum)" clear-value="clearCounter">
+                    <!-- Icons -->
+                    <i class="dropdown-select-icon fa fa-lg" ng-class="currentMethod.icon"></i>
+
+                    <!-- Dropdown menu -->
+                    <ul class="dropdown-select-menu pull-right" role="menu">
+                      <li ng-repeat="method in methods">
+                        <a href="javascript:void(0)" ng-click="setMethod(method)">
+                          <i class="fa fa-lg" ng-class="method.icon"></i> {{ method.title }}
+                        </a>
+                      </li>
+                    </ul>
                   </div>
-                </div>
+                </td>
-              </td>
+              </tr>
-            </tr>
 
-            <tr ng-if="currentMethod.id == 'webhook'">
+              <tr ng-repeat="field in currentMethod.fields">
-              <td colspan="2">
+                <td valign="top" class="name">{{ field.title }}:</td>
-                <div class="alert alert-info" style="margin-top: 20px; margin-bottom: 0px">
+                <td>
-                  JSON metadata representing the event will be <b>POST</b>ed to the URL.
+                  <div ng-switch on="field.type">
-                  <br><br>
+                    <span ng-switch-when="email">
-                  The contents for each event can be found in the user guide:
+                      <input type="email" class="form-control" ng-model="currentConfig[field.name]" required>
-                  <a href="http://docs.quay.io/guides/notifications.html#webhook{{ currentEvent.id ? '_' + currentEvent.id : '' }}"
+                    </span>
-                     target="_blank">
+                    <input type="url" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="url" required>
-                    http://docs.quay.io/guides/notifications.html
+                    <input type="text" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="string" required>
-                  </a>
+                    <!-- TODO(jschorr): unify the ability to create an input box with all the usual features -->
-                </div>
+                    <div ng-switch-when="regex">
-              </td>
+                      <input type="text" class="form-control" ng-model="currentConfig[field.name]"
-            </tr>
+                             ng-pattern="getPattern(field)"
-          </table>
+                             placeholder="{{ field.placeholder }}"
+                             ng-name="field.name"
+                             id="{{ field.name }}"
+                             required>
+
+                      <div class="alert alert-warning" style="margin-top: 10px; margin-bottom: 10px"
+                            ng-if="field.regex_fail_message && hasRegexMismatch(createForm.$error, field.name)">
+                        <span ng-bind-html="field.regex_fail_message"></span>
+                      </div>
+                    </div>
+                    <div class="entity-search" namespace="repository.namespace"
+                         placeholder="''"
+                         current-entity="currentConfig[field.name]"
+                         ng-model="currentConfig[field.name]"
+                         allowed-entities="['user', 'team', 'org']"
+                         ng-switch-when="entity"></div>
+
+                    <div ng-if="getHelpUrl(field, currentConfig)"
+                         style="margin-top: 10px;  margin-bottom: 10px">
+                      See: <a href="{{ getHelpUrl(field, currentConfig) }}" target="_blank">{{ getHelpUrl(field, currentConfig) }}</a>
+                    </div>
+
+                    <div class="co-alert co-alert-info" ng-if="currentMethod.id == 'webhook'"
+                         style="margin-top: 6px; margin-bottom: 0px">
+                      JSON metadata representing the event will be <b>POST</b>ed to the URL.
+                      <br><br>
+                      The contents for each event can be found in the user guide:
+                      <a href="http://docs.quay.io/guides/notifications.html#webhook{{ currentEvent.id ? '_' + currentEvent.id : '' }}"
+                         target="_blank">
+                        http://docs.quay.io/guides/notifications.html
+                      </a>
+                    </div>
+                  </div>
+                </td>
+              </tr>
+            </table>
+          </div>
        </div>
 
         <!-- Auth e-mail button bar -->

static/js/directives/object-order-by.js

@@ -0,0 +1,17 @@
+// From: http://justinklemm.com/angularjs-filter-ordering-objects-ngrepeat/ under MIT License
+quayApp.filter('orderObjectBy', function() {
+  return function(items, field, reverse) {
+    var filtered = [];
+    angular.forEach(items, function(item) {
+      filtered.push(item);
+    });
+
+    filtered.sort(function (a, b) {
+      return (a[field] > b[field] ? 1 : -1);
+    });
+
+    if(reverse) filtered.reverse();
+
+    return filtered;
+  };
+});

static/js/directives/ui/create-external-notification-dialog.js

@@ -18,6 +18,7 @@ angular.module('quay').directive('createExternalNotificationDialog', function ()
       $scope.currentMethod = null;
       $scope.status = '';
       $scope.currentConfig = {};
+      $scope.currentEventConfig = {};
       $scope.clearCounter = 0;
       $scope.unauthorizedEmail = false;
 
@@ -30,6 +31,7 @@ angular.module('quay').directive('createExternalNotificationDialog', function ()
 
       $scope.setEvent = function(event) {
         $scope.currentEvent = event;
+        $scope.currentEventConfig = {};
       };
 
       $scope.setMethod = function(method) {
@@ -89,6 +91,7 @@ angular.module('quay').directive('createExternalNotificationDialog', function ()
           'event': $scope.currentEvent.id,
           'method': $scope.currentMethod.id,
           'config': $scope.currentConfig,
+          'eventConfig': $scope.currentEventConfig,
           'title': $scope.currentTitle
         };
 

static/js/services/external-notification-data.js

@@ -2,9 +2,9 @@
  * Service which defines the various kinds of external notification and provides methods for
  * easily looking up information about those kinds.
  */
-angular.module('quay').factory('ExternalNotificationData', ['Config', 'Features',
+angular.module('quay').factory('ExternalNotificationData', ['Config', 'Features','VulnerabilityService',
 
-function(Config, Features) {
+function(Config, Features, VulnerabilityService) {
   var externalNotificationData = {};
 
   var events = [
@@ -43,6 +43,22 @@ function(Config, Features) {
     }
   }
 
+  if (Features.SECURITY_SCANNER) {
+    events.push({
+      'id': 'vulnerability_found',
+      'title': 'Package Vulnerability Found',
+      'icon': 'fa-flag',
+      'fields': [
+        {
+          'name': 'level',
+          'type': 'enum',
+          'title': 'Minimum Severity Level',
+          'values': VulnerabilityService.LEVELS,
+        }
+      ]
+    });
+  }
+
   var methods = [
     {
       'id': 'quay_notification',

static/js/services/notification-service.js

@@ -3,9 +3,9 @@
  * in the sidebar) and provides helper methods for working with them.
  */
 angular.module('quay').factory('NotificationService',
-  ['$rootScope', '$interval', 'UserService', 'ApiService', 'StringBuilderService', 'PlanService', 'UserService', 'Config', '$location',
+  ['$rootScope', '$interval', 'UserService', 'ApiService', 'StringBuilderService', 'PlanService', 'UserService', 'Config', '$location', 'VulnerabilityService',
 
-function($rootScope, $interval, UserService, ApiService, StringBuilderService, PlanService, UserService, Config, $location) {
+function($rootScope, $interval, UserService, ApiService, StringBuilderService, PlanService, UserService, Config, $location, VulnerabilityService) {
   var notificationService = {
     'user': null,
     'notifications': [],
@@ -120,6 +120,16 @@ function($rootScope, $interval, UserService, ApiService, StringBuilderService, P
         return '/repository/' + metadata.repository + '/build?current=' + metadata.build_id;
       },
       'dismissable': true
+    },
+    'vulnerability_found': {
+      'level': function(metadata) {
+        var priority = metadata['vulnerability']['priority'];
+        return VulnerabilityService.LEVELS[priority].level;
+      },
+      'message': 'A {vulnerability.priority} vulnerability was detected in repository {repository}',
+      'page': function(metadata) {
+        return '/repository/' + metadata.repository + '?tab=tags';
+      }
     }
   };
 
@@ -182,7 +192,13 @@ function($rootScope, $interval, UserService, ApiService, StringBuilderService, P
     if (!kindInfo) {
       return 'notification-info';
     }
-    return 'notification-' + kindInfo['level'];
+
+    var level = kindInfo['level'];
+    if (level != null && typeof level != 'string') {
+      level = level(notification['metadata']);
+    }
+
+    return 'notification-' + level;
   };
 
   notificationService.getClasses = function(notifications) {

static/js/services/string-builder-service.js

@@ -4,6 +4,27 @@
 angular.module('quay').factory('StringBuilderService', ['$sce', 'UtilService', function($sce, UtilService) {
   var stringBuilderService = {};
 
+  var fieldIcons = {
+    'inviter': 'user',
+    'username': 'user',
+    'user': 'user',
+    'email': 'envelope',
+    'activating_username': 'user',
+    'delegate_user': 'user',
+    'delegate_team': 'group',
+    'team': 'group',
+    'token': 'key',
+    'repo': 'hdd-o',
+    'robot': 'ci-robot',
+    'tag': 'tag',
+    'role': 'th-large',
+    'original_role': 'th-large',
+    'application_name': 'cloud',
+    'image': 'archive',
+    'original_image': 'archive',
+    'client_id': 'chain'
+  };
+
   stringBuilderService.buildUrl = function(value_or_func, metadata) {
     var url = value_or_func;
     if (typeof url != 'string') {
@@ -43,28 +64,47 @@ angular.module('quay').factory('StringBuilderService', ['$sce', 'UtilService', f
     return $sce.trustAsHtml(stringBuilderService.buildString(value_or_func, metadata, opt_codetag));
   };
 
-  stringBuilderService.buildString = function(value_or_func, metadata, opt_codetag) {
+  stringBuilderService.replaceField = function(description, prefix, key, value, opt_codetag) {
-     var fieldIcons = {
+    if (Array.isArray(value)) {
-      'inviter': 'user',
+      value = value.join(', ');
-      'username': 'user',
+    } else if (typeof value == 'object') {
-      'user': 'user',
+      for (var subkey in value) {
-      'email': 'envelope',
+        if (value.hasOwnProperty(subkey)) {
-      'activating_username': 'user',
+          description = stringBuilderService.replaceField(description, prefix + key + '.',
-      'delegate_user': 'user',
+            subkey, value[subkey], opt_codetag)
-      'delegate_team': 'group',
+        }
-      'team': 'group',
+      }
-      'token': 'key',
-      'repo': 'hdd-o',
-      'robot': 'ci-robot',
-      'tag': 'tag',
-      'role': 'th-large',
-      'original_role': 'th-large',
-      'application_name': 'cloud',
-      'image': 'archive',
-      'original_image': 'archive',
-      'client_id': 'chain'
-    };
 
+      return description
+    }
+
+    value = value.toString();
+
+    if (key.indexOf('image') >= 0) {
+      value = value.substr(0, 12);
+    }
+
+    var safe = UtilService.escapeHtmlString(value);
+    var markedDown = UtilService.getMarkedDown(value);
+    markedDown = markedDown.substr('<p>'.length, markedDown.length - '<p></p>'.length);
+
+    var icon = fieldIcons[key];
+    if (icon) {
+      if (icon.indexOf('ci-') < 0) {
+        icon = 'fa-' + icon;
+      }
+
+      markedDown = '<i class="fa ' + icon + '"></i>' + markedDown;
+    }
+
+    var codeTag = opt_codetag || 'code';
+    description = description.replace('{' + prefix + key + '}',
+      '<' + codeTag + ' title="' + safe  + '">' + markedDown + '</' + codeTag + '>');
+
+    return description
+  }
+
+  stringBuilderService.buildString = function(value_or_func, metadata, opt_codetag) {
     var filters = {
       'obj': function(value) {
         if (!value) { return []; }
@@ -89,32 +129,7 @@ angular.module('quay').factory('StringBuilderService', ['$sce', 'UtilService', f
           value = filters[key](value);
         }
 
-        if (Array.isArray(value)) {
+        description = stringBuilderService.replaceField(description, '', key, value, opt_codetag);
-          value = value.join(', ');
-        }
-
-        value = value.toString();
-
-        if (key.indexOf('image') >= 0) {
-          value = value.substr(0, 12);
-        }
-
-        var safe = UtilService.escapeHtmlString(value);
-        var markedDown = UtilService.getMarkedDown(value);
-        markedDown = markedDown.substr('<p>'.length, markedDown.length - '<p></p>'.length);
-
-        var icon = fieldIcons[key];
-        if (icon) {
-          if (icon.indexOf('ci-') < 0) {
-            icon = 'fa-' + icon;
-          }
-
-          markedDown = '<i class="fa ' + icon + '"></i>' + markedDown;
-        }
-
-        var codeTag = opt_codetag || 'code';
-        description = description.replace('{' + key + '}',
-          '<' + codeTag + ' title="' + safe  + '">' + markedDown + '</' + codeTag + '>');
       }
     }
     return description.replace('\n', '<br>');

static/js/services/vulnerability-service.js

@@ -0,0 +1,98 @@
+/**
+ * Service which provides helper methods for working with the vulnerability system.
+  */
+angular.module('quay').factory('VulnerabilityService', ['Config', function(Config) {
+  var vulnService = {};
+
+  // NOTE: This objects are used directly in the external-notification-data service, so make sure
+  // to update that code if the format here is changed.
+  vulnService.LEVELS = {
+    'Unknown': {
+      'title': 'Unknown',
+      'index': '6',
+      'level': 'info',
+
+      'description': 'Unknown is either a security problem that has not been assigned ' +
+                     'to a priority yet or a priority that our system did not recognize',
+      'banner_required': false
+    },
+
+    'Negligible': {
+      'title': 'Negligible',
+      'index': '5',
+      'level': 'info',
+
+      'description': 'Negligible is technically a security problem, but is only theoretical ' +
+                     'in nature, requires a very special situation, has almost no install base, ' +
+                     'or does no real damage.',
+      'banner_required': false
+    },
+
+    'Low': {
+      'title': 'Low',
+      'index': '4',
+      'level': 'warning',
+
+      'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
+                     'requires a user-assisted attack, a small install base, or does very ' +
+                     'little damage.',
+      'banner_required': false
+    },
+
+    'Medium': {
+      'title': 'Medium',
+      'value': 'Medium',
+      'index': '3',
+      'level': 'warning',
+
+      'description': 'Medium is a real security problem, and is exploitable for many people. ' +
+                     'Includes network daemon denial of service attacks, cross-site scripting, ' +
+                     'and gaining user privileges.',
+      'banner_required': false
+    },
+
+    'High': {
+      'title': 'High',
+      'value': 'High',
+      'index': '2',
+      'level': 'warning',
+
+      'description': 'High is a real problem, exploitable for many people in a default installation. ' +
+                     'Includes serious remote denial of services, local root privilege escalations, ' +
+                     'or data loss.',
+      'banner_required': false
+    },
+
+    'Critical': {
+      'title': 'Critical',
+      'value': 'Critical',
+      'index': '1',
+      'level': 'error',
+
+      'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
+                     'a installation of the package. Includes remote root privilege escalations, ' +
+                     'or massive data loss.',
+      'banner_required': true
+    },
+
+    'Defcon1': {
+      'title': 'Defcon 1',
+      'value': 'Defcon1',
+      'index': '0',
+      'level': 'error',
+
+      'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +
+                     'by the Quay team. It requires immediate attention.',
+      'banner_required': true
+    }
+  };
+
+  vulnService.getLevels = function() {
+    return Object.keys(vulnService.LEVELS).map(function(key) {
+      return vulnService.LEVELS[key];
+    });
+  };
+
+  return vulnService;
+}]);
+

test/test_api_security.py

@@ -50,6 +50,8 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
                                      SuperUserOrganizationManagement, SuperUserOrganizationList,
                                      SuperUserAggregateLogs)
 
+from endpoints.api.secscan import RepositoryImagePackages, RepositoryTagVulnerabilities
+
 
 try:
   app.register_blueprint(api_bp, url_prefix='/api')
@@ -4210,18 +4212,54 @@ class TestOrganizationInvoiceField(ApiTestCase):
     ApiTestCase.setUp(self)
     self._set_url(OrganizationInvoiceField, orgname='buynlarge', field_uuid='1234')
 
-  def test_get_anonymous(self):
+  def test_delete_anonymous(self):
     self._run_test('DELETE', 403, None, None)
 
-  def test_get_freshuser(self):
+  def test_delete_freshuser(self):
     self._run_test('DELETE', 403, 'freshuser', None)
 
-  def test_get_reader(self):
+  def test_delete_reader(self):
     self._run_test('DELETE', 403, 'reader', None)
 
-  def test_get_devtable(self):
+  def test_delete_devtable(self):
     self._run_test('DELETE', 201, 'devtable', None)
 
 
+class TestRepositoryTagVulnerabilities(ApiTestCase):
+  def setUp(self):
+    ApiTestCase.setUp(self)
+    self._set_url(RepositoryTagVulnerabilities, repository='devtable/simple', tag='latest')
+
+  def test_get_anonymous(self):
+    self._run_test('GET', 401, None, None)
+
+  def test_get_freshuser(self):
+    self._run_test('GET', 403, 'freshuser', None)
+
+  def test_get_reader(self):
+    self._run_test('GET', 403, 'reader', None)
+
+  def test_get_devtable(self):
+    self._run_test('GET', 200, 'devtable', None)
+
+
+class TestRepositoryImagePackages(ApiTestCase):
+  def setUp(self):
+    ApiTestCase.setUp(self)
+    self._set_url(RepositoryImagePackages, repository='devtable/simple', imageid='fake')
+
+  def test_get_anonymous(self):
+    self._run_test('GET', 401, None, None)
+
+  def test_get_freshuser(self):
+    self._run_test('GET', 403, 'freshuser', None)
+
+  def test_get_reader(self):
+    self._run_test('GET', 403, 'reader', None)
+
+  def test_get_devtable(self):
+    self._run_test('GET', 404, 'devtable', None)
+
+
 if __name__ == '__main__':
   unittest.main()

test/test_api_usage.py

@@ -1549,8 +1549,8 @@ class TestDeleteRepository(ApiTestCase):
     model.build.create_repository_build(repository, delegate_token, {}, 'someid2', 'foobar2')
 
     # Create some notifications.
-    model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {})
+    model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {}, {})
-    model.notification.create_repo_notification(repository, 'build_queued', 'slack', {})
+    model.notification.create_repo_notification(repository, 'build_queued', 'slack', {}, {})
 
     # Create some logs.
     model.log.log_action('push_repo', ADMIN_ACCESS_USER, repository=repository)
@@ -1984,7 +1984,7 @@ class TestRepositoryNotifications(ApiTestCase):
     json = self.postJsonResponse(RepositoryNotificationList,
                                  params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
                                  data=dict(config={'url': 'http://example.com'}, event='repo_push',
-                                           method='webhook'),
+                                           method='webhook', eventConfig={}),
                                  expected_code=201)
 
     self.assertEquals('repo_push', json['event'])
@@ -2024,7 +2024,8 @@ class TestRepositoryNotifications(ApiTestCase):
     json = self.postJsonResponse(RepositoryNotificationList,
                                  params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
                                  data=dict(config={'url': 'http://example.com'}, event='repo_push',
-                                           method='webhook', title='Some Notification'),
+                                           method='webhook', title='Some Notification',
+                                           eventConfig={}),
                                  expected_code=201)
 
     self.assertEquals('repo_push', json['event'])

test/testconfig.py

@@ -55,3 +55,10 @@ class TestConfig(DefaultConfig):
   FEATURE_GITHUB_BUILD = True
 
   CLOUDWATCH_NAMESPACE = None
+
+  FEATURE_SECURITY_SCANNER = True
+  SECURITY_SCANNER = {
+    'ENDPOINT': 'http://localhost/some/invalid/path',
+    'ENGINE_VERSION_TARGET': 1,
+    'API_CALL_TIMEOUT': 1
+  }

util/migrate/backfill_parent_id.py

@@ -0,0 +1,49 @@
+import logging
+from data.database import Image, ImageStorage, db
+from app import app
+
+logger = logging.getLogger(__name__)
+
+def backfill_parent_id():
+  logger.setLevel(logging.DEBUG)
+
+  logger.debug('backfill_parent_id: Starting')
+  logger.debug('backfill_parent_id: This can be a LONG RUNNING OPERATION. Please wait!')
+
+  # Check for any images without parent
+  has_images = bool(list(Image
+                         .select(Image.id)
+                         .join(ImageStorage)
+                         .where(Image.parent >> None, Image.ancestors != '/', ImageStorage.uploading == False)
+                         .limit(1)))
+
+  if not has_images:
+    logger.debug('backfill_parent_id: No migration needed')
+    return
+
+  while True:
+    # Load the record from the DB.
+    batch_images_ids = list(Image
+                            .select(Image.id)
+                            .join(ImageStorage)
+                            .where(Image.parent >> None, Image.ancestors != '/', ImageStorage.uploading == False)
+                            .limit(100))
+
+    if len(batch_images_ids) == 0:
+      logger.debug('backfill_parent_id: Completed')
+      return
+
+    for image_id in batch_images_ids:
+      with app.config['DB_TRANSACTION_FACTORY'](db):
+        try:
+          image = Image.select(Image.id, Image.ancestors).where(Image.id == image_id).get()
+          image.parent = image.ancestors.split('/')[-2]
+          image.save()
+        except Image.DoesNotExist:
+          pass
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.DEBUG)
+  logging.getLogger('peewee').setLevel(logging.CRITICAL)
+
+  backfill_parent_id()

util/migrate/migrategithubdeploykeys.py

@@ -2,7 +2,8 @@ import logging
 import logging.config
 import json
 
-from data.database import RepositoryBuildTrigger, BuildTriggerService, db, db_for_update
+from data.database import (db, db_for_update, BaseModel, CharField, ForeignKeyField,
+                           TextField, BooleanField)
 from app import app
 from buildtrigger.basehandler import BuildTriggerHandler
 from util.security.ssh import generate_ssh_keypair
@@ -10,6 +11,32 @@ from github import GithubException
 
 logger = logging.getLogger(__name__)
 
+class BuildTriggerService(BaseModel):
+  name = CharField(index=True, unique=True)
+
+class Repository(BaseModel):
+  pass
+
+class User(BaseModel):
+  pass
+
+class AccessToken(BaseModel):
+  pass
+
+class RepositoryBuildTrigger(BaseModel):
+  uuid = CharField()
+  service = ForeignKeyField(BuildTriggerService, index=True)
+  repository = ForeignKeyField(Repository, index=True)
+  connected_user = ForeignKeyField(User)
+  auth_token = CharField(null=True)
+  private_key = TextField(null=True)
+  config = TextField(default='{}')
+  write_token = ForeignKeyField(AccessToken, null=True)
+  pull_robot = ForeignKeyField(User, null=True, related_name='triggerpullrobot')
+
+  used_legacy_github = BooleanField(null=True, default=False)
+
+
 def backfill_github_deploykeys():
   """ Generates and saves private deploy keys for any GitHub build triggers still relying on
       the old buildpack behavior. """

util/secscan/secscanendpoint.py

@@ -0,0 +1,50 @@
+import features
+import logging
+import requests
+import json
+
+from urlparse import urljoin
+
+logger = logging.getLogger(__name__)
+
+class SecurityScanEndpoint(object):
+  """ Helper class for talking to the Security Scan service (Clair). """
+  def __init__(self, app, config_provider):
+    self.app = app
+    self.config_provider = config_provider
+
+    if not features.SECURITY_SCANNER:
+      return
+
+    self.security_config = app.config['SECURITY_SCANNER']
+
+    self.certificate = self._getfilepath('CA_CERTIFICATE_FILENAME') or False
+    self.public_key = self._getfilepath('PUBLIC_KEY_FILENAME')
+    self.private_key = self._getfilepath('PRIVATE_KEY_FILENAME')
+
+    if self.public_key and self.private_key:
+      self.keys = (self.public_key, self.private_key)
+    else:
+      self.keys = None
+
+  def _getfilepath(self, config_key):
+    security_config = self.security_config
+
+    if config_key in security_config:
+      with self.config_provider.get_volume_file(security_config[config_key]) as f:
+        return f.name
+
+    return None
+
+  def call_api(self, relative_url, *args, **kwargs):
+    """ Issues an HTTP call to the sec API at the given relative URL. """
+    security_config = self.security_config
+    api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
+    url = urljoin(api_url, relative_url % args)
+
+    client = self.app.config['HTTPCLIENT']
+    timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
+    logger.debug('Looking up sec information: %s', url)
+
+    return client.get(url, params=kwargs, timeout=timeout, cert=self.keys,
+                      verify=self.certificate)

workers/securityworker.py

@@ -0,0 +1,217 @@
+import logging
+import requests
+import features
+import time
+import os
+import random
+
+from sys import exc_info
+from peewee import JOIN_LEFT_OUTER
+from app import app, storage, OVERRIDE_CONFIG_DIRECTORY
+from workers.worker import Worker
+from data.database import Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement, db_random_func, UseThenDisconnect
+
+logger = logging.getLogger(__name__)
+
+BATCH_SIZE = 20
+INDEXING_INTERVAL = 10
+API_METHOD_INSERT = '/layers'
+API_METHOD_VERSION = '/versions/engine'
+
+def _get_image_to_export(version):
+  Parent = Image.alias()
+  ParentImageStorage = ImageStorage.alias()
+  rimages = []
+
+  # Without parent
+  candidates = (Image
+    .select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum)
+    .join(ImageStorage)
+    .where(Image.security_indexed_engine < version, Image.parent >> None, ImageStorage.uploading == False, ImageStorage.checksum != '')
+    .limit(BATCH_SIZE*10)
+    .alias('candidates'))
+
+  images = (Image
+    .select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum)
+    .from_(candidates)
+    .order_by(db_random_func())
+    .tuples()
+    .limit(BATCH_SIZE))
+
+  for image in images:
+    rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': None, 'parent_storage_uuid': None})
+
+  # With analyzed parent
+  candidates = (Image
+      .select(Image.docker_image_id, ImageStorage.uuid, ImageStorage.checksum, Parent.docker_image_id.alias('parent_docker_image_id'), ParentImageStorage.uuid.alias('parent_storage_uuid'))
+      .join(Parent, on=(Image.parent == Parent.id))
+      .join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage))
+      .switch(Image)
+      .join(ImageStorage)
+      .where(Image.security_indexed_engine < version, Parent.security_indexed == True, Parent.security_indexed_engine >= version, ImageStorage.uploading == False, ImageStorage.checksum != '')
+      .limit(BATCH_SIZE*10)
+      .alias('candidates'))
+
+  images = (Image
+    .select(candidates.c.docker_image_id, candidates.c.uuid, candidates.c.checksum, candidates.c.parent_docker_image_id, candidates.c.parent_storage_uuid)
+    .from_(candidates)
+    .order_by(db_random_func())
+    .tuples()
+    .limit(BATCH_SIZE))
+
+  for image in images:
+    rimages.append({'docker_image_id': image[0], 'storage_uuid': image[1], 'storage_checksum': image[2], 'parent_docker_image_id': image[3], 'parent_storage_uuid': image[4]})
+
+  # Re-shuffle, otherwise the images without parents will always be on the top
+  random.shuffle(rimages)
+
+  return rimages
+
+def _get_storage_locations(uuid):
+  query = (ImageStoragePlacement
+      .select()
+      .join(ImageStorageLocation)
+      .switch(ImageStoragePlacement)
+      .join(ImageStorage, JOIN_LEFT_OUTER)
+      .where(ImageStorage.uuid == uuid))
+
+  locations = list()
+  for location in query:
+    locations.append(location.location.name)
+
+  return locations
+
+def _update_image(image, indexed, version):
+  query = (Image
+      .select()
+      .join(ImageStorage)
+      .where(Image.docker_image_id == image['docker_image_id'], ImageStorage.uuid == image['storage_uuid']))
+
+  updated_images = list()
+  for image in query:
+    updated_images.append(image.id)
+
+  query = (Image
+      .update(security_indexed=indexed, security_indexed_engine=version)
+      .where(Image.id << updated_images))
+  query.execute()
+
+class SecurityWorker(Worker):
+  def __init__(self):
+    super(SecurityWorker, self).__init__()
+    if self._load_configuration():
+      self.add_operation(self._index_images, INDEXING_INTERVAL)
+
+  def _load_configuration(self):
+    # Load configuration
+    config = app.config.get('SECURITY_SCANNER')
+
+    if not config or not 'ENDPOINT' in config or not 'ENGINE_VERSION_TARGET' in config or not 'DISTRIBUTED_STORAGE_PREFERENCE' in app.config:
+      logger.exception('No configuration found for the security worker')
+      return False
+    self._api = config['ENDPOINT']
+    self._target_version = config['ENGINE_VERSION_TARGET']
+    self._default_storage_locations = app.config['DISTRIBUTED_STORAGE_PREFERENCE']
+
+    self._ca_verification = False
+    self._cert = None
+    if 'CA_CERTIFICATE_FILENAME' in config:
+      self._ca_verification = os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['CA_CERTIFICATE_FILENAME'])
+      if not os.path.isfile(self._ca_verification):
+        logger.exception('Could not find configured CA file')
+        return False
+    if 'PRIVATE_KEY_FILENAME' in config and 'PUBLIC_KEY_FILENAME' in config:
+      self._cert = (
+        os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['PUBLIC_KEY_FILENAME']),
+        os.path.join(OVERRIDE_CONFIG_DIRECTORY, config['PRIVATE_KEY_FILENAME']),
+      )
+      if not os.path.isfile(self._cert[0]) or not os.path.isfile(self._cert[1]):
+        logger.exception('Could not find configured key pair files')
+        return False
+
+    return True
+
+  def _index_images(self):
+    with UseThenDisconnect(app.config):
+      while True:
+        # Get images to analyze
+        try:
+          images = _get_image_to_export(self._target_version)
+        except Image.DoesNotExist:
+          logger.debug('No more image to analyze')
+          return
+
+        for img in images:
+          # Get layer storage URL
+          path = storage.image_layer_path(img['storage_uuid'])
+          locations = self._default_storage_locations
+          if not storage.exists(locations, path):
+            locations = _get_storage_locations(img['storage_uuid'])
+          if not storage.exists(locations, path):
+            logger.warning('Could not find a valid location to download layer %s', img['docker_image_id']+'.'+img['storage_uuid'])
+            # Mark as analyzed because that error is most likely to occur during the pre-process, with the database copy
+            # when images are actually removed on the real database (and therefore in S3)
+            _update_image(img, False, self._target_version)
+            continue
+          uri = storage.get_direct_download_url(locations, path)
+          if uri == None:
+            # Local storage hack
+            uri = path
+
+          # Forge request
+          request = {
+            'ID': img['docker_image_id']+'.'+img['storage_uuid'],
+            'TarSum': img['storage_checksum'],
+            'Path': uri
+          }
+
+          if img['parent_docker_image_id'] is not None and img['parent_storage_uuid'] is not None:
+            request['ParentID'] = img['parent_docker_image_id']+'.'+img['parent_storage_uuid']
+
+          # Post request
+          try:
+            logger.info('Analyzing %s', request['ID'])
+            # Using invalid certificates doesn't return proper errors because of
+            # https://github.com/shazow/urllib3/issues/556
+            httpResponse = requests.post(self._api + API_METHOD_INSERT, json=request, cert=self._cert, verify=self._ca_verification)
+          except:
+            logger.exception('An exception occurred when analyzing layer ID %s : %s', request['ID'], exc_info()[0])
+            return
+          try:
+            jsonResponse = httpResponse.json()
+          except:
+            logger.exception('An exception occurred when analyzing layer ID %s : the response is not valid JSON (%s)', request['ID'], httpResponse.text)
+            return
+
+          if httpResponse.status_code == 201:
+            # The layer has been successfully indexed
+            api_version = jsonResponse['Version']
+            if api_version < self._target_version:
+              logger.warning('An engine runs on version %d but the target version is %d')
+            _update_image(img, True, api_version)
+            logger.info('Layer ID %s : analyzed successfully', request['ID'])
+          else:
+            if 'Message' in jsonResponse:
+              if 'OS and/or package manager are not supported' in jsonResponse['Message']:
+                # The current engine could not index this layer
+                logger.warning('A warning event occurred when analyzing layer ID %s : %s', request['ID'], jsonResponse['Message'])
+                # Hopefully, there is no version lower than the target one running
+                _update_image(img, False, self._target_version)
+              else:
+                logger.exception('An exception occurred when analyzing layer ID %s : %d %s', request['ID'], httpResponse.status_code, jsonResponse['Message'])
+                return
+            else:
+              logger.exception('An exception occurred when analyzing layer ID %s : %d', request['ID'], httpResponse.status_code)
+              return
+
+if __name__ == '__main__':
+  logging.getLogger('requests').setLevel(logging.WARNING)
+  logging.getLogger('apscheduler').setLevel(logging.CRITICAL)
+
+  if not features.SECURITY_SCANNER:
+    logger.debug('Security scanner disabled; skipping')
+    while True:
+      time.sleep(100000)
+
+  worker = SecurityWorker()
+  worker.start()