Ready for demo

data/model/image.py

@@ -1,7 +1,7 @@
 import logging
 import dateutil.parser
 
-from peewee import JOIN_LEFT_OUTER, fn
+from peewee import JOIN_LEFT_OUTER, fn, SQL
 from datetime import datetime
 
 from data.model import DataModelException, db_transaction, _basequery, storage
@@ -18,7 +18,8 @@ def get_repository_images_recursive(docker_image_ids):
 
       Note: This is a DB intensive operation and should be used sparingly.
   """
-  inner_images = Image.select('%/' + Image.id + '/%').where(Image.docker_image_id << docker_image_ids)
+  # TODO: test this on MySQL and Postgres
+  inner_images = Image.select(SQL('"%/" || id || "/%"')).where(Image.docker_image_id << docker_image_ids)
 
   images = Image.select(Image.id).where(Image.docker_image_id << docker_image_ids)
   recursive_images = Image.select(Image.id).where(Image.ancestors ** inner_images)

endpoints/api/sec.py

@@ -65,7 +65,8 @@ class RepositoryTagVulnerabilities(RepositoryParamResource):
         'security_indexed': False
       }
 
-    data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id,
+    data = _call_security_api('layers/%s.%s/vulnerabilities', tag_image.docker_image_id,
+                              tag_image.storage.uuid,
                               minimumPriority=args.minimumPriority)
 
     return {
@@ -79,7 +80,7 @@ class RepositoryTagVulnerabilities(RepositoryParamResource):
 @path_param('repository', 'The full path of the repository. e.g. namespace/name')
 @path_param('imageid', 'The image ID')
 class RepositoryImagePackages(RepositoryParamResource):
-  """ Operations for listing the packages added/removed in an image. """
+  """ Operations for listing the packages in an image. """
 
   @require_repo_read
   @nickname('getRepoImagePackages')
@@ -94,7 +95,8 @@ class RepositoryImagePackages(RepositoryParamResource):
         'security_indexed': False
       }
 
-    data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id)
+    data = _call_security_api('layers/%s.%s/packages', repo_image.docker_image_id,
+                              repo_image.storage.uuid)
 
     return {
       'security_indexed': True,

endpoints/sec.py

@@ -13,14 +13,17 @@ sec = Blueprint('sec', __name__)
 @sec.route('/notification', methods=['POST'])
 def sec_notification():
   data = request.get_json()
-  print data
 
   # Find all tags that contain the layer(s) introducing the vulnerability.
   # TODO: remove this check once fixed.
   if not 'IntroducingLayersIDs' in data['Content']:
     return make_response('Okay')
 
-  layer_ids = data['Content']['IntroducingLayersIDs']
+  # TODO: fix this for the image_id.storage thing properly.
+  layer_ids = [full_id.split('.')[0] for full_id in data['Content']['IntroducingLayersIDs']]
+  if not layer_ids:
+    return make_response('Okay')
+
   tags = model.tag.get_matching_tags(layer_ids, RepositoryTag, Repository, Image)
 
   # For any repository that has a notification setup, issue a notification.

static/css/directives/repo-view/repo-panel-tags.css

@@ -94,8 +94,9 @@
 }
 
 .repo-panel-tags-element .vuln-description {
-  color: #ccc;
+  color: #aaa;
   font-size: 10px;
+  white-space: normal;
 }
 
 .repo-panel-tags-element .fa-flag.None {
@@ -110,6 +111,10 @@
   color: red;
 }
 
+.repo-panel-tags-element .vuln-dropdown ul {
+  min-width: 400px;
+}
+
 @keyframes flickerAnimation { /* flame pulses */
   0%   { opacity:1; }
   50%  { opacity:0; }

static/directives/repo-view/repo-panel-tags.html

@@ -126,7 +126,7 @@
              data-title="Image has no vulnerabilities"
              bs-tooltip>
           </i>
-          <div class="dropdown" style="text-align: left;"
+          <div class="dropdown vuln-dropdown" style="text-align: left;"
                ng-if="getTagVulnerabilities(tag).indexed && getTagVulnerabilities(tag).hasVulnerabilities">
             <i class="fa fa-flag"
                data-title="Image has vulnerabilities"

static/partials/image-view.html

@@ -68,21 +68,21 @@
                 Please try again in a few minutes.
               </div>
             </div>
-            <div class="empty" ng-if="packages.security_indexed && !packages.data.InstalledPackages.length">
+            <div class="empty" ng-if="packages.security_indexed && !packages.data.Packages.length">
               <div class="empty-primary-msg">This image contains no recognized packages</div>
               <div class="empty-secondary-msg">
                 Quay currently indexes Debian, Red Hat and Ubuntu packages.
               </div>
             </div>
 
-            <table class="table" ng-if="packages.security_indexed && packages.data.InstalledPackages.length">
+            <table class="table" ng-if="packages.security_indexed && packages.data.Packages.length">
               <thead>
                 <th>Package Name</th>
                 <th>Package Version</th>
                 <th>OS</th>
               </thead>
 
-              <tr ng-repeat="package in packages.data.InstalledPackages | orderBy:'Name'">
+              <tr ng-repeat="package in packages.data.Packages | orderBy:'Name'">
                 <td>{{ package.Name }}</td>
                 <td>{{ package.Version }}</td>
                 <td>{{ package.OS }}</td>

workers/securityworker.py

@@ -97,13 +97,13 @@ def _update_image(image, indexed, version):
       .where(Image.docker_image_id == image['docker_image_id'], ImageStorage.uuid == image['storage_uuid']))
 
   updated_images = list()
-  for image in query:
+  for row in query:
-    updated_images.append(image.id)
+    updated_images.append(row.id)
 
-  query = (Image
+  (Image
       .update(security_indexed=indexed, security_indexed_engine=version)
-      .where(Image.id << updated_images))
+      .where(Image.id << updated_images)
-  query.execute()
+      .execute())
 
 class SecurityWorker(Worker):
   def __init__(self):
@@ -163,14 +163,17 @@ class SecurityWorker(Worker):
           # Get layer storage URL
           path = storage.image_layer_path(img['storage_uuid'])
           locations = self._default_storage_locations
+
           if not storage.exists(locations, path):
             locations = _get_storage_locations(img['storage_uuid'])
+
           if not storage.exists(locations, path):
             logger.warning('Could not find a valid location to download layer %s', img['docker_image_id']+'.'+img['storage_uuid'])
             # Mark as analyzed because that error is most likely to occur during the pre-process, with the database copy
             # when images are actually removed on the real database (and therefore in S3)
             _update_image(img, False, self._target_version)
             continue
+
           uri = storage.get_direct_download_url(locations, path)
           if uri == None:
             # Local storage hack
@@ -200,69 +203,7 @@ class SecurityWorker(Worker):
             logger.exception('An exception occurred when analyzing layer ID %s : the response is not valid JSON (%s)', request['ID'], httpResponse.text)
             return
 
-          if httpResponse.status_code == 201:
+          if httpResponse.status_code != 201:
-            # The layer has been successfully indexed
-            api_version = jsonResponse['Version']
-            if api_version < self._target_version:
-              logger.warning('An engine runs on version %d but the target version is %d')
-            _update_image(img, True, api_version)
-            logger.info('Layer ID %s : analyzed successfully', request['ID'])
-
-
-            # TODO(jschorr): Put this in a proper place, properly comment, unify with the
-            # callback code, etc.
-            try:
-              logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
-              response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
-            except requests.exceptions.Timeout:
-              logger.debug('Timeout when calling Sec')
-              continue
-            except requests.exceptions.ConnectionError:
-              logger.debug('Connection error when calling Sec')
-              continue
-
-            logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
-            if response.status_code == 404:
-              continue
-
-            sec_data = json.loads(response.text)
-            logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
-
-            if not sec_data['Vulnerabilities']:
-              continue
-
-            event = ExternalNotificationEvent.get(name='vulnerability_found')
-            matching = (RepositoryTag
-              .select(RepositoryTag, Repository)
-              .distinct()
-              .join(Repository)
-              .join(RepositoryNotification)
-              .where(RepositoryNotification.event == event,
-                     RepositoryTag.image == img['image_id']))
-
-            repository_map = defaultdict(list)
-
-            for tag in matching:
-              repository_map[tag.repository_id].append(tag)
-
-            for repository_id in repository_map:
-              tags = repository_map[repository_id]
-
-              for vuln in sec_data['Vulnerabilities']:
-                event_data = {
-                  'tags': [tag.name for tag in tags],
-                  'vulnerability': {
-                    'id': vuln['ID'],
-                    'description': vuln['Description'],
-                    'link': vuln['Link'],
-                    'priority': vuln['Priority'],
-                  },
-                }
-
-                spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
-
-
-          else:
             if 'Message' in jsonResponse:
               if 'OS and/or package manager are not supported' in jsonResponse['Message']:
                 # The current engine could not index this layer
@@ -276,6 +217,68 @@ class SecurityWorker(Worker):
               logger.exception('An exception occurred when analyzing layer ID %s : %d', request['ID'], httpResponse.status_code)
               return
 
+          # The layer has been successfully indexed
+          api_version = jsonResponse['Version']
+          if api_version < self._target_version:
+            logger.warning('An engine runs on version %d but the target version is %d')
+
+          logger.debug('Layer %s analyzed successfully', request['ID'])
+          _update_image(img, True, api_version)
+
+
+          # TODO(jschorr): Put this in a proper place, properly comment, unify with the
+          # callback code, etc.
+          try:
+            logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
+            response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
+          except requests.exceptions.Timeout:
+            logger.debug('Timeout when calling Sec')
+            continue
+          except requests.exceptions.ConnectionError:
+            logger.debug('Connection error when calling Sec')
+            continue
+
+          logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
+          if response.status_code == 404:
+            continue
+
+          sec_data = json.loads(response.text)
+          logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
+
+          if not sec_data['Vulnerabilities']:
+            continue
+
+          event = ExternalNotificationEvent.get(name='vulnerability_found')
+          matching = (RepositoryTag
+            .select(RepositoryTag, Repository)
+            .distinct()
+            .join(Repository)
+            .join(RepositoryNotification)
+            .where(RepositoryNotification.event == event,
+                   RepositoryTag.image == img['image_id']))
+
+          repository_map = defaultdict(list)
+
+          for tag in matching:
+            repository_map[tag.repository_id].append(tag)
+
+          for repository_id in repository_map:
+            tags = repository_map[repository_id]
+
+            for vuln in sec_data['Vulnerabilities']:
+              event_data = {
+                'tags': [tag.name for tag in tags],
+                'vulnerability': {
+                  'id': vuln['ID'],
+                  'description': vuln['Description'],
+                  'link': vuln['Link'],
+                  'priority': vuln['Priority'],
+                },
+              }
+
+              spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
+
+
 if __name__ == '__main__':
   if not features.SECURITY_SCANNER:
     logger.debug('Security scanner disabled; skipping')