From c8d825c2327072100c92dca2fe8aa89b60754368 Mon Sep 17 00:00:00 2001
From: Quentin Machu <me@quentin-machu.fr>
Date: Tue, 16 Feb 2016 15:16:22 -0500
Subject: [PATCH 01/12] expose min_id in allocator.py
---
util/migrate/allocator.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/util/migrate/allocator.py b/util/migrate/allocator.py
index c0502a843..bf998e1dd 100644
--- a/util/migrate/allocator.py
+++ b/util/migrate/allocator.py
@@ -14,9 +14,9 @@ class NoAvailableKeysError(ValueError):
class CompletedKeys(object):
- def __init__(self, max_index):
+ def __init__(self, min_index, max_index):
self._max_index = max_index
- self._min_index = 0
+ self._min_index = min_index
self._slabs = RBTree()
def _get_previous_or_none(self, index):
@@ -118,7 +118,7 @@ class CompletedKeys(object):
return random.randint(hole_start, rand_max_bound)
-def yield_random_entries(batch_query, primary_key_field, batch_size, max_id):
+def yield_random_entries(batch_query, primary_key_field, batch_size, max_id, min_id=0):
""" This method will yield items from random blocks in the database. We will track metadata
about which keys are available for work, and we will complete the backfill when there is no
more work to be done. The method yields tupes of (candidate, Event), and if the work was
@@ -126,8 +126,9 @@ def yield_random_entries(batch_query, primary_key_field, batch_size, max_id):
an "id" field which can be inspected.
"""
+ min_id = max(min_id, 0)
max_id = max(max_id, 1)
- allocator = CompletedKeys(max_id + 1)
+ allocator = CompletedKeys(min_id, max_id + 1)
try:
while True:
From e5da33578c24667ef9d0f1bcbea96e755320a590 Mon Sep 17 00:00:00 2001
From: Quentin Machu <me@quentin-machu.fr>
Date: Wed, 17 Feb 2016 14:44:49 -0500
Subject: [PATCH 02/12] Adapt security worker for Clair v1.0 (except
notifications)
---
data/model/image.py | 68 ++--------
workers/securityworker.py | 262 +++++++++++++++++++++-----------------
2 files changed, 160 insertions(+), 170 deletions(-)
diff --git a/data/model/image.py b/data/model/image.py
index 0665c5750..8078abd58 100644
--- a/data/model/image.py
+++ b/data/model/image.py
@@ -426,57 +426,16 @@ def ensure_image_locations(*names):
data = [{'name': name} for name in insert_names]
ImageStorageLocation.insert_many(data).execute()
-def get_secscan_candidates(engine_version, batch_size):
+def get_image_with_storage_and_parent_base():
Parent = Image.alias()
ParentImageStorage = ImageStorage.alias()
- rimages = []
-
- # Collect the images without parents.
- candidates = list(Image
- .select(Image.id)
- .join(ImageStorage)
- .where(Image.security_indexed_engine < engine_version,
- Image.parent >> None,
- ImageStorage.uploading == False)
- .limit(batch_size*10))
-
- if len(candidates) > 0:
- images = (Image
- .select(Image, ImageStorage)
- .join(ImageStorage)
- .where(Image.id << candidates)
- .order_by(db_random_func())
- .limit(batch_size))
- rimages.extend(images)
-
- # Collect the images with analyzed parents.
- candidates = list(Image
- .select(Image.id)
- .join(Parent, on=(Image.parent == Parent.id))
- .switch(Image)
- .join(ImageStorage)
- .where(Image.security_indexed_engine < engine_version,
- Parent.security_indexed_engine == engine_version,
- ImageStorage.uploading == False)
- .limit(batch_size*10))
-
- if len(candidates) > 0:
- images = (Image
- .select(Image, ImageStorage, Parent, ParentImageStorage)
- .join(Parent, on=(Image.parent == Parent.id))
- .join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage))
- .switch(Image)
- .join(ImageStorage)
- .where(Image.id << candidates)
- .order_by(db_random_func())
- .limit(batch_size))
- rimages.extend(images)
-
- # Shuffle the images, otherwise the images without parents will always be on the top
- random.shuffle(rimages)
-
- return rimages
+ return (Image
+ .select(Image, ImageStorage, Parent, ParentImageStorage)
+ .join(ImageStorage)
+ .switch(Image)
+ .join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id))
+ .join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)))
def set_secscan_status(image, indexed, version):
query = (Image
@@ -487,12 +446,13 @@ def set_secscan_status(image, indexed, version):
ids_to_update = [row.id for row in query]
if not ids_to_update:
- return
+ return False
- (Image
- .update(security_indexed=indexed, security_indexed_engine=version)
- .where(Image.id << ids_to_update)
- .execute())
+ return (Image
+ .update(security_indexed=indexed, security_indexed_engine=version)
+ .where(Image.id << ids_to_update)
+ .where((Image.security_indexed_engine != version) | (Image.security_indexed != indexed))
+ .execute()) != 0
def find_or_create_derived_storage(source_image, transformation_name, preferred_location):
@@ -536,5 +496,3 @@ def delete_derived_storage_by_uuid(storage_uuid):
return
image_storage.delete_instance(recursive=True)
-
-
diff --git a/workers/securityworker.py b/workers/securityworker.py
index e6fe492a1..51998e068 100644
--- a/workers/securityworker.py
+++ b/workers/securityworker.py
@@ -5,24 +5,26 @@ import requests
import features
import time
-from endpoints.notificationhelper import spawn_notification
+from peewee import fn
from collections import defaultdict
+
from app import app, config_provider, storage, secscan_api
+from endpoints.notificationhelper import spawn_notification
from workers.worker import Worker
from data import model
+from data.database import (Image, UseThenDisconnect, ExternalNotificationEvent)
from data.model.tag import filter_tags_have_repository_event, get_tags_for_image
-from data.model.image import get_secscan_candidates, set_secscan_status
+from data.model.image import set_secscan_status, get_image_with_storage_and_parent_base
from data.model.storage import get_storage_locations
-from data.database import ExternalNotificationEvent
from util.secscan.api import SecurityConfigValidator
-
-logger = logging.getLogger(__name__)
+from util.migrate.allocator import yield_random_entries
BATCH_SIZE = 50
INDEXING_INTERVAL = 30
API_METHOD_INSERT = '/v1/layers'
-API_METHOD_VERSION = '/v1/versions/engine'
+API_METHOD_GET_WITH_VULNERABILITIES = '/v1/layers/%s?vulnerabilities'
+logger = logging.getLogger(__name__)
class SecurityWorker(Worker):
def __init__(self):
@@ -40,6 +42,26 @@ class SecurityWorker(Worker):
else:
logger.warning('Failed to validate security scan configuration')
+ def _new_request(self, image):
+ """ Create the request body to submit the given image for analysis. """
+ url = self._get_image_url(image)
+ if url is None:
+ return None
+
+ request = {
+ 'Layer': {
+ 'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid),
+ 'Path': url,
+ 'Format': 'Docker'
+ }
+ }
+
+ if image.parent.docker_image_id and image.parent.storage.uuid:
+ request['Layer']['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
+ image.parent.storage.uuid)
+
+ return request
+
def _get_image_url(self, image):
""" Gets the download URL for an image and if the storage doesn't exist,
marks the image as unindexed. """
@@ -71,147 +93,157 @@ class SecurityWorker(Worker):
return uri
- def _new_request(self, image):
- url = self._get_image_url(image)
- if url is None:
- return None
+ def _index_images(self):
+ def batch_query():
+ base_query = get_image_with_storage_and_parent_base()
+ return base_query.where(Image.security_indexed_engine < self._target_version)
- request = {
- 'ID': '%s.%s' % (image.docker_image_id, image.storage.uuid),
- 'Path': url,
- }
+ min_id = (Image
+ .select(fn.Min(Image.id))
+ .where(Image.security_indexed_engine < self._target_version)
+ .scalar())
+ max_id = Image.select(fn.Max(Image.id)).scalar()
- if image.parent is not None:
- request['ParentID'] = '%s.%s' % (image.parent.docker_image_id,
- image.parent.storage.uuid)
+ with UseThenDisconnect(app.config):
+ for candidate, abt in yield_random_entries(batch_query, Image.id, BATCH_SIZE, max_id, min_id):
+ _, continue_batch = self._analyze_recursively(candidate)
+ if not continue_batch:
+ logger.info('Another worker pre-empted us for layer: %s', candidate.id)
+ abt.set()
- return request
+ def _analyze_recursively(self, layer):
+ """ Analyzes a layer and all its parents """
+ if layer.parent_id and layer.parent.security_indexed_engine < self._target_version:
+ # The image has a parent that is not analyzed yet with this engine.
+ # Get the parent to get it's own parent and recurse.
+ try:
+ base_query = get_image_with_storage_and_parent_base()
+ parent_layer = base_query.where(Image.id == layer.parent_id).get()
+ except Image.DoesNotExist:
+ logger.warning("Image %s has Image %s as parent but doesn't exist.", layer.id,
+ layer.parent_id)
- def _analyze_image(self, image):
- """ Analyzes an image by passing it to Clair. """
- request = self._new_request(image)
+ return False, set_secscan_status(layer, False, self._target_version)
+
+ cont, _ = self._analyze_recursively(parent_layer)
+ if not cont:
+ # The analysis failed for some reason and did not mark the layer as failed,
+ # thus we should not try to analyze the children of that layer.
+ # Interrupt the recursive analysis and return as no-one pre-empted us.
+ return False, True
+
+ # Now we know all parents are analyzed.
+ return self._analyze(layer)
+
+ def _analyze(self, layer):
+ """ Analyzes a single layer.
+ Return two bools, the first one tells us if we should evaluate its children, the second
+ one is set to False when another worker pre-empted the candidate's analysis for us. """
+
+ # If the parent couldn't be analyzed with the target version or higher, we can't analyze
+ # this image. Mark it as failed with the current target version.
+ if (layer.parent_id and not layer.parent.security_indexed and
+ layer.parent.security_indexed_engine >= self._target_version):
+ return True, set_secscan_status(layer, False, self._target_version)
+
+ request = self._new_request(layer)
if request is None:
- return False
+ return False, True
# Analyze the image.
try:
- logger.info('Analyzing %s', request['ID'])
+ logger.info('Analyzing layer %s', request['Layer']['Name'])
# Using invalid certificates doesn't return proper errors because of
# https://github.com/shazow/urllib3/issues/556
- httpResponse = requests.post(self._api + API_METHOD_INSERT, json=request,
- cert=self._keys, verify=self._cert)
- jsonResponse = httpResponse.json()
+ http_response = requests.post(self._api + API_METHOD_INSERT, json=request,
+ cert=self._keys, verify=self._cert)
+ json_response = http_response.json()
except (requests.exceptions.RequestException, ValueError):
- logger.exception('An exception occurred when analyzing layer ID %s', request['ID'])
- return False
+ logger.exception('An exception occurred when analyzing layer %s', request['Layer']['Name'])
+ return False, True
# Handle any errors from the security scanner.
- if httpResponse.status_code != 201:
- message = jsonResponse.get('Message', '')
- if 'OS and/or package manager are not supported' in message or 'could not extract' in message:
- # The current engine could not index this layer or we tried to index a manifest.
- logger.warning('A warning event occurred when analyzing layer ID %s : %s',
- request['ID'], jsonResponse['Message'])
+ if http_response.status_code != 201:
+ message = json_response.get('Error').get('Message', '')
+ logger.warning('A warning event occurred when analyzing layer %s (status code %s): %s',
+ request['Layer']['Name'], http_response.status_code, message)
- # Hopefully, there is no version lower than the target one running
- set_secscan_status(image, False, self._target_version)
-
- return False
+ # 422 means that the layer could not be analyzed:
+ # - the layer could not be extracted (manifest?)
+ # - the layer operating system / package manager is unsupported
+ # Set the layer as failed.
+ if http_response.status_code == 422:
+ return True, set_secscan_status(layer, False, self._target_version)
else:
- logger.warning('Got non-201 when analyzing layer ID %s: %s', request['ID'], jsonResponse)
- return False
+ return False, True
# Verify that the version matches.
- api_version = jsonResponse['Version']
+ api_version = json_response['Layer']['IndexedByVersion']
if api_version < self._target_version:
- logger.warning('An engine runs on version %d but the target version is %d')
+ logger.warning('An engine runs on version %d but the target version is %d', api_version,
+ self._target_version)
# Mark the image as analyzed.
- logger.debug('Layer %s analyzed successfully', image.id)
- set_secscan_status(image, True, api_version)
+ logger.info('Analyzed layer %s successfully', request['Layer']['Name'])
+ set_status = set_secscan_status(layer, True, api_version)
- return True
+ # If we are the one who've done the job successfully first, get the vulnerabilities and
+ # send notifications to the repos that have a tag on that layer.
+ # TODO(josephschorr): Adapt this depending on the new notification format we adopt.
+ # if set_status:
+ # # Get the tags of the layer we analyzed.
+ # repository_map = defaultdict(list)
+ # event = ExternalNotificationEvent.get(name='vulnerability_found')
+ # matching = list(filter_tags_have_repository_event(get_tags_for_image(layer.id), event))
+ #
+ # for tag in matching:
+ # repository_map[tag.repository_id].append(tag)
+ #
+ # # If there is at least one tag,
+ # # Lookup the vulnerabilities for the image, now that it is analyzed.
+ # if len(repository_map) > 0:
+ # logger.debug('Loading vulnerabilities for layer %s', layer.id)
+ # sec_data = self._get_vulnerabilities(layer)
+ #
+ # if sec_data is not None:
+ # # Dispatch events for any detected vulnerabilities
+ # logger.debug('Got vulnerabilities for layer %s: %s', layer.id, sec_data)
+ #
+ # for repository_id in repository_map:
+ # tags = repository_map[repository_id]
+ #
+ # for vuln in sec_data['Vulnerabilities']:
+ # event_data = {
+ # 'tags': [tag.name for tag in tags],
+ # 'vulnerability': {
+ # 'id': vuln['Name'],
+ # 'description': vuln['Description'],
+ # 'link': vuln['Link'],
+ # 'priority': vuln['Priority'],
+ # },
+ # }
+ #
+ # spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
- def _get_vulnerabilities(self, image):
+ return True, set_status
+
+ def _get_vulnerabilities(self, layer):
""" Returns the vulnerabilities detected (if any) or None on error. """
try:
- response = secscan_api.call('layers/%s/vulnerabilities', None,
- '%s.%s' % (image.docker_image_id, image.storage.uuid))
+ response = secscan_api.call(self._api + API_METHOD_GET_WITH_VULNERABILITIES, None,
+ '%s.%s' % (layer.docker_image_id, layer.storage.uuid))
+
logger.debug('Got response %s for vulnerabilities for layer %s',
- response.status_code, image.id)
+ response.status_code, layer.id)
+
if response.status_code == 404:
return None
except (requests.exceptions.RequestException, ValueError):
- logger.exception('Failed to get vulnerability response for %s', image.id)
+ logger.exception('Failed to get vulnerability response for %s', layer.id)
return None
return response.json()
- def _index_images(self):
- logger.debug('Started indexing')
- event = ExternalNotificationEvent.get(name='vulnerability_found')
-
- while True:
- # Lookup the images to index.
- images = []
- logger.debug('Looking up images to index')
- images = get_secscan_candidates(self._target_version, BATCH_SIZE)
-
- if not images:
- logger.debug('No more images left to analyze')
- return
-
- logger.debug('Found %d images to index', len(images))
- for image in images:
- # If we couldn't analyze the parent, we can't analyze this image.
- if (image.parent and not image.parent.security_indexed and
- image.parent.security_indexed_engine >= self._target_version):
- set_secscan_status(image, False, self._target_version)
- continue
-
- # Analyze the image.
- analyzed = self._analyze_image(image)
- if not analyzed:
- continue
-
- # Get the tags of the image we analyzed
- matching = list(filter_tags_have_repository_event(get_tags_for_image(image.id), event))
-
- repository_map = defaultdict(list)
-
- for tag in matching:
- repository_map[tag.repository_id].append(tag)
-
- # If there is at least one tag,
- # Lookup the vulnerabilities for the image, now that it is analyzed.
- if len(repository_map) > 0:
- logger.debug('Loading vulnerabilities for layer %s', image.id)
- sec_data = self._get_vulnerabilities(image)
-
- if sec_data is None:
- continue
-
- if not sec_data.get('Vulnerabilities'):
- continue
-
- # Dispatch events for any detected vulnerabilities
- logger.debug('Got vulnerabilities for layer %s: %s', image.id, sec_data)
-
- for repository_id in repository_map:
- tags = repository_map[repository_id]
-
- for vuln in sec_data['Vulnerabilities']:
- event_data = {
- 'tags': [tag.name for tag in tags],
- 'vulnerability': {
- 'id': vuln['ID'],
- 'description': vuln['Description'],
- 'link': vuln['Link'],
- 'priority': vuln['Priority'],
- },
- }
-
- spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
if __name__ == '__main__':
if not features.SECURITY_SCANNER:
From 4bd5996bbf1f5244f26dd389c2c670575c5cca09 Mon Sep 17 00:00:00 2001
From: Quentin Machu <me@quentin-machu.fr>
Date: Wed, 17 Feb 2016 14:48:50 -0500
Subject: [PATCH 03/12] Adapt secscan API for Clair v1.0
Squash /vulnerabilities and /packages as it basically does the same
action on Clair and we don't need both for Quay
---
endpoints/api/secscan.py | 52 ++++++++++------------------------------
1 file changed, 12 insertions(+), 40 deletions(-)
diff --git a/endpoints/api/secscan.py b/endpoints/api/secscan.py
index 91a5ee891..146227a76 100644
--- a/endpoints/api/secscan.py
+++ b/endpoints/api/secscan.py
@@ -1,4 +1,4 @@
-""" List and manage repository vulnerabilities and other sec information. """
+""" List and manage repository vulnerabilities and other security information. """
import logging
import features
@@ -9,7 +9,7 @@ from app import secscan_api
from data import model
from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
RepositoryParamResource, resource, nickname, show_if, parse_args,
- query_param)
+ query_param, truthy_bool)
logger = logging.getLogger(__name__)
@@ -54,19 +54,19 @@ def _get_status(repo_image):
@show_if(features.SECURITY_SCANNER)
-@resource('/v1/repository/<apirepopath:repository>/image/<imageid>/vulnerabilities')
+@resource('/v1/repository/<apirepopath:repository>/image/<imageid>/security')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('imageid', 'The image ID')
-class RepositoryImageVulnerabilities(RepositoryParamResource):
+class RepositoryImageSecurity(RepositoryParamResource):
""" Operations for managing the vulnerabilities in a repository image. """
@require_repo_read
- @nickname('getRepoImageVulnerabilities')
+ @nickname('getRepoImageSecurity')
@parse_args()
- @query_param('minimumPriority', 'Minimum vulnerability priority', type=str,
- default='Low')
+ @query_param('vulnerabilities', 'Include vulnerabilities informations', type=truthy_bool,
+ default=False)
def get(self, namespace, repository, imageid, parsed_args):
- """ Fetches the vulnerabilities (if any) for a repository tag. """
+ """ Fetches the features and vulnerabilities (if any) for a repository tag. """
repo_image = model.image.get_repo_image(namespace, repository, imageid)
if repo_image is None:
raise NotFound()
@@ -79,40 +79,12 @@ class RepositoryImageVulnerabilities(RepositoryParamResource):
}
layer_id = '%s.%s' % (repo_image.docker_image_id, repo_image.storage.uuid)
- data = _call_security_api('layers/%s/vulnerabilities', layer_id,
- minimumPriority=parsed_args.minimumPriority)
+ if parsed_args.vulnerabilities:
+ data = _call_security_api('layers/%s?vulnerabilities', layer_id)
+ else:
+ data = _call_security_api('layers/%s?features', layer_id)
return {
'status': _get_status(repo_image),
'data': data,
}
-
-
-@show_if(features.SECURITY_SCANNER)
-@resource('/v1/repository/<apirepopath:repository>/image/<imageid>/packages')
-@path_param('repository', 'The full path of the repository. e.g. namespace/name')
-@path_param('imageid', 'The image ID')
-class RepositoryImagePackages(RepositoryParamResource):
- """ Operations for listing the packages added/removed in an image. """
-
- @require_repo_read
- @nickname('getRepoImagePackages')
- def get(self, namespace, repository, imageid):
- """ Fetches the packages added/removed in the given repo image. """
- repo_image = model.image.get_repo_image(namespace, repository, imageid)
- if repo_image is None:
- raise NotFound()
-
- if not repo_image.security_indexed:
- return {
- 'status': _get_status(repo_image),
- }
-
- layer_id = '%s.%s' % (repo_image.docker_image_id, repo_image.storage.uuid)
- data = _call_security_api('layers/%s/packages', layer_id)
-
- return {
- 'status': _get_status(repo_image),
- 'data': data,
- }
-
From 6a37f937185677cb54c8ff5c534af97a510b6aa3 Mon Sep 17 00:00:00 2001
From: Quentin Machu <me@quentin-machu.fr>
Date: Wed, 17 Feb 2016 14:50:37 -0500
Subject: [PATCH 04/12] Make the Quay UI work for Clair 1.0
---
.../directives/repo-view/repo-panel-tags.js | 50 ++++++++------
static/js/pages/image-view.js | 66 +++++++++++-------
static/partials/image-view.html | 67 +++++++++++--------
3 files changed, 106 insertions(+), 77 deletions(-)
diff --git a/static/js/directives/repo-view/repo-panel-tags.js b/static/js/directives/repo-view/repo-panel-tags.js
index ddce687bf..21e061826 100644
--- a/static/js/directives/repo-view/repo-panel-tags.js
+++ b/static/js/directives/repo-view/repo-panel-tags.js
@@ -156,41 +156,48 @@ angular.module('quay').directive('repoPanelTags', function () {
var params = {
'imageid': image_id,
'repository': $scope.repository.namespace + '/' + $scope.repository.name,
+ 'vulnerabilities': true,
};
- ApiService.getRepoImageVulnerabilities(null, params).then(function(resp) {
+ ApiService.getRepoImageSecurity(null, params).then(function(resp) {
imageData.loading = false;
imageData.status = resp['status'];
if (imageData.status == 'scanned') {
- var vulnerabilities = resp.data.Vulnerabilities;
-
- imageData.hasVulnerabilities = !!vulnerabilities.length;
- imageData.vulnerabilities = vulnerabilities;
-
+ var vulnerabilities = [];
var highest = {
- 'Priority': 'Unknown',
+ 'Severity': 'Unknown',
'Count': 0,
'index': 100000
};
- resp.data.Vulnerabilities.forEach(function(v) {
- if (VulnerabilityService.LEVELS[v.Priority].index == 0) {
- $scope.defcon1[v.ID] = v;
- $scope.hasDefcon1 = true;
- }
+ if (resp.data && resp.data.Layer && resp.data.Layer.Features) {
+ resp.data.Layer.Features.forEach(function(feature) {
+ if (feature.Vulnerabilities) {
+ feature.Vulnerabilities.forEach(function(vuln) {
+ if (VulnerabilityService.LEVELS[vuln.Severity].index == 0) {
+ $scope.defcon1[vuln.ID] = v;
+ $scope.hasDefcon1 = true;
+ }
- if (VulnerabilityService.LEVELS[v.Priority].index < highest.index) {
- highest = {
- 'Priority': v.Priority,
- 'Count': 1,
- 'index': VulnerabilityService.LEVELS[v.Priority].index
+ if (VulnerabilityService.LEVELS[vuln.Severity].index < highest.index) {
+ highest = {
+ 'Priority': vuln.Severity,
+ 'Count': 1,
+ 'index': VulnerabilityService.LEVELS[vuln.Severity].index
+ }
+ } else if (VulnerabilityService.LEVELS[vuln.Severity].index == highest.index) {
+ highest['Count']++;
+ }
+
+ vulnerabilities.push(vuln);
+ });
}
- } else if (VulnerabilityService.LEVELS[v.Priority].index == highest.index) {
- highest['Count']++;
- }
- });
+ });
+ }
+ imageData.hasVulnerabilities = !!vulnerabilities.length;
+ imageData.vulnerabilities = vulnerabilities;
imageData.highestVulnerability = highest;
}
}, function() {
@@ -355,4 +362,3 @@ angular.module('quay').directive('repoPanelTags', function () {
};
return directiveDefinitionObject;
});
-
diff --git a/static/js/pages/image-view.js b/static/js/pages/image-view.js
index 253c389b2..5e377eac0 100644
--- a/static/js/pages/image-view.js
+++ b/static/js/pages/image-view.js
@@ -45,39 +45,53 @@
loadImage();
loadRepository();
- $scope.downloadPackages = function() {
- if (!Features.SECURITY_SCANNER || $scope.packagesResource) { return; }
-
- var params = {
- 'repository': namespace + '/' + name,
- 'imageid': imageid
- };
-
- $scope.packagesResource = ApiService.getRepoImagePackagesAsResource(params).get(function(packages) {
- $scope.packages = packages;
- return packages;
- });
- };
-
- $scope.loadImageVulnerabilities = function() {
- if (!Features.SECURITY_SCANNER || $scope.vulnerabilitiesResource) { return; }
+ $scope.loadImageSecurity = function() {
+ if (!Features.SECURITY_SCANNER || $scope.securityResource) { return; }
$scope.VulnerabilityLevels = VulnerabilityService.getLevels();
var params = {
'repository': namespace + '/' + name,
- 'imageid': imageid
+ 'imageid': imageid,
+ 'vulnerabilities': true,
};
- $scope.vulnerabilitiesResource = ApiService.getRepoImageVulnerabilitiesAsResource(params).get(function(resp) {
- $scope.vulnerabilityInfo = resp;
- $scope.vulnerabilities = [];
+ $scope.securityResource = ApiService.getRepoImageSecurityAsResource(params).get(function(resp) {
+ $scope.securityStatus = resp.status;
+ $scope.securityFeatures = [];
+ $scope.securityVulnerabilities = [];
- if (resp.data && resp.data.Vulnerabilities) {
- resp.data.Vulnerabilities.forEach(function(vuln) {
- vuln_copy = jQuery.extend({}, vuln);
- vuln_copy['index'] = VulnerabilityService.LEVELS[vuln['Priority']]['index'];
- $scope.vulnerabilities.push(vuln_copy);
+ if (resp.data && resp.data.Layer && resp.data.Layer.Features) {
+ resp.data.Layer.Features.forEach(function(feature) {
+ feature_obj = {
+ 'name': feature.Name,
+ 'namespace': feature.Namespace,
+ 'version': feature.Version,
+ 'addedBy': feature.AddedBy,
+ }
+ feature_vulnerabilities = []
+
+ if (feature.Vulnerabilities) {
+ feature.Vulnerabilities.forEach(function(vuln) {
+ vuln_obj = {
+ 'name': vuln.Name,
+ 'namespace': vuln.Namespace,
+ 'description': vuln.Description,
+ 'link': vuln.Link,
+ 'severity': vuln.Severity,
+ 'metadata': vuln.Metadata,
+ 'feature': jQuery.extend({}, feature_obj),
+ 'fixedBy': vuln.FixedBy,
+ 'index': VulnerabilityService.LEVELS[vuln['Severity']]['index'],
+ }
+
+ feature_vulnerabilities.push(vuln_obj)
+ $scope.securityVulnerabilities.push(vuln_obj);
+ });
+ }
+
+ feature_obj['vulnerabilities'] = feature_vulnerabilities
+ $scope.securityFeatures.push(feature_obj);
});
}
@@ -94,4 +108,4 @@
}, 100);
};
}
-})();
\ No newline at end of file
+})();
diff --git a/static/partials/image-view.html b/static/partials/image-view.html
index 3cd8627a3..b3343802f 100644
--- a/static/partials/image-view.html
+++ b/static/partials/image-view.html
@@ -21,13 +21,13 @@
<span class="cor-tab" tab-active="true" tab-title="Layers" tab-target="#layers">
<i class="fa ci-layers"></i>
</span>
- <span class="cor-tab" tab-title="Security Scan" tab-target="#security"
- tab-init="loadImageVulnerabilities()"
+ <span class="cor-tab" tab-title="Security Scan" tab-target="#vulnerabilities"
+ tab-init="loadImageSecurity()"
quay-show="Features.SECURITY_SCANNER">
<i class="fa fa-bug"></i>
</span>
<span class="cor-tab" tab-title="Packages" tab-target="#packages"
- tab-init="downloadPackages()"
+ tab-init="loadImageSecurity()"
quay-show="Features.SECURITY_SCANNER">
<i class="fa ci-package"></i>
</span>
@@ -42,51 +42,58 @@
ng-repeat="parent in reversedHistory"></div>
</div>
- <!-- Security -->
- <div id="security" class="tab-pane" quay-require="['SECURITY_SCANNER']">
- <div class="resource-view" resource="vulnerabilitiesResource" error-message="'Could not load security information for image'">
+ <!-- Vulnerabilities -->
+ <div id="vulnerabilities" class="tab-pane" quay-require="['SECURITY_SCANNER']">
+ <div class="resource-view" resource="securityResource" error-message="'Could not load security information for image'">
<div class="col-md-9">
- <div class="filter-box floating" collection="vulnerabilities" filter-model="options.vulnFilter" filter-name="Vulnerabilities" ng-if="vulnerabilityInfo.status == 'scanned' && vulnerabilities.length"></div>
+ <div class="filter-box floating" collection="securityVulnerabilities" filter-model="options.vulnFilter" filter-name="Vulnerabilities" ng-if="securityStatus == 'scanned' && securityVulnerabilities.length"></div>
<h3>Image Security</h3>
- <div class="empty" ng-if="vulnerabilityInfo.status == 'queued'">
+ <div class="empty" ng-if="securityStatus == 'queued'">
<div class="empty-primary-msg">This image has not been indexed yet</div>
<div class="empty-secondary-msg">
Please try again in a few minutes.
</div>
</div>
- <div class="empty" ng-if="vulnerabilityInfo.status == 'failed'">
+ <div class="empty" ng-if="securityStatus == 'failed'">
<div class="empty-primary-msg">This image could not be indexed</div>
<div class="empty-secondary-msg">
Our security scanner was unable to index this image.
</div>
</div>
- <div class="empty" ng-if="vulnerabilityInfo.status == 'scanned' && !vulnerabilities.length">
+ <div class="empty" ng-if="securityStatus == 'scanned' && !securityVulnerabilities.length">
<div class="empty-primary-msg">This image contains no recognized security vulnerabilities</div>
<div class="empty-secondary-msg">
- Quay currently indexes Debian, Red Hat and Ubuntu packages.
+ Quay currently indexes Debian, Red Hat and Ubuntu based images.
</div>
</div>
- <div ng-if="vulnerabilityInfo.status == 'scanned' && vulnerabilities.length">
+ <div ng-if="securityStatus == 'scanned' && securityVulnerabilities.length">
<table class="co-table">
<thead>
<td style="width: 200px;">Vulnerability</td>
<td style="width: 200px;">Priority</td>
+ <td style="width: 200px;">Introduced by</td>
+ <td style="width: 200px;">Fixed by</td>
<td>Description</td>
</thead>
- <tr ng-repeat="vulnerability in vulnerabilities | filter:options.vulnFilter | orderBy:'index'">
- <td><a href="{{ vulnerability.Link }}" target="_blank">{{ vulnerability.ID }}</a></td>
+ <tr ng-repeat="vulnerability in securityVulnerabilities | filter:options.vulnFilter | orderBy:'index'">
+ <td><a href="{{ vulnerability.link }}" target="_blank">{{ vulnerability.name }}</a></td>
<td>
- <span class="vulnerability-priority-view" priority="vulnerability.Priority"></span>
- <td>{{ vulnerability.Description }}</td>
+ <span class="vulnerability-priority-view" priority="vulnerability.severity"></span>
+ </td>
+ <td>{{ vulnerability.feature.name }} {{ vulnerability.feature.version }}</td>
+ <td>
+ <span ng-if="vulnerability.fixedBy">{{ vulnerability.feature.name }} {{ vulnerability.fixedBy }}</span>
+ </td>
+ <td>{{ vulnerability.description }}</td>
</tr>
</table>
- <div class="empty" ng-if="(vulnerabilities | filter:options.vulnFilter).length == 0"
+ <div class="empty" ng-if="(securityVulnerabilities | filter:options.vulnFilter).length == 0"
style="margin-top: 20px;">
<div class="empty-primary-msg">No matching vulnerabilities found</div>
<div class="empty-secondary-msg">
@@ -110,41 +117,43 @@
</div>
</div>
- <!-- Packages -->
+ <!-- Features -->
<div id="packages" class="tab-pane" quay-require="['SECURITY_SCANNER']">
- <div class="resource-view" resource="packagesResource" error-message="'Could not load image packages'">
- <div class="filter-box floating" collection="packages.data.Packages" filter-model="options.packageFilter" filter-name="Packages" ng-if="packages.status == 'scanned' && packages.data.Packages.length"></div>
+ <div class="resource-view" resource="securityResource" error-message="'Could not load image packages'">
+ <div class="filter-box floating" collection="securityFeatures" filter-model="options.packageFilter" filter-name="Features" ng-if="securityStatus == 'scanned' && securityFeatures.length"></div>
<h3>Image Packages</h3>
- <div class="empty" ng-if="packages.status == 'queued'">
+ <div class="empty" ng-if="securityStatus == 'queued'">
<div class="empty-primary-msg">This image has not been indexed yet</div>
<div class="empty-secondary-msg">
Please try again in a few minutes.
</div>
</div>
- <div class="empty" ng-if="packages.status == 'failed'">
+ <div class="empty" ng-if="securityStatus == 'failed'">
<div class="empty-primary-msg">This image could not be indexed</div>
<div class="empty-secondary-msg">
Our security scanner was unable to index this image.
</div>
</div>
- <table class="co-table" ng-if="packages.status == 'scanned'">
+ <table class="co-table" ng-if="securityStatus == 'scanned'">
<thead>
<td>Package Name</td>
<td>Package Version</td>
- <td>OS</td>
+ <td>Package OS</td>
+ <td>Number of vulnerabilities</td>
</thead>
- <tr ng-repeat="package in packages.data.Packages | filter:options.packageFilter | orderBy:'Name'">
- <td>{{ package.Name }}</td>
- <td>{{ package.Version }}</td>
- <td>{{ package.OS }}</td>
+ <tr ng-repeat="feature in securityFeatures | filter:options.packageFilter | orderBy:'Name'">
+ <td>{{ feature.name }}</td>
+ <td>{{ feature.version }}</td>
+ <td>{{ feature.namespace }}</td>
+ <td>{{ feature.vulnerabilities.length }}</td>
</tr>
</table>
- <div class="empty" ng-if="(packages.data.Packages | filter:options.packageFilter).length == 0"
+ <div class="empty" ng-if="(securityFeatures | filter:options.packageFilter).length == 0"
style="margin-top: 20px;">
<div class="empty-primary-msg">No matching packages found</div>
<div class="empty-secondary-msg" ng-if="options.packageFilter">
From c0374d71c94fadf835cc0b218d9fd1e97efd45f9 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <josephschorr@users.noreply.github.com>
Date: Wed, 24 Feb 2016 16:01:27 -0500
Subject: [PATCH 05/12] Refactor the security worker and API calls and add a
bunch of tests
---
app.py | 3 +-
data/database.py | 3 +
endpoints/api/secscan.py | 42 ++---
endpoints/common.py | 3 +-
endpoints/notificationevent.py | 2 +-
initdb.py | 46 ++---
requirements-nover.txt | 3 +-
requirements.txt | 1 +
test/test_api_security.py | 24 +--
test/test_api_usage.py | 71 ++++++++
test/test_secscan.py | 242 +++++++++++++++++++++++++
test/testconfig.py | 5 +-
util/secscan/__init__.py | 90 +++++++++
util/secscan/analyzer.py | 138 ++++++++++++++
util/secscan/api.py | 321 +++++++++++++++------------------
util/secscan/validator.py | 65 +++++++
workers/securityworker.py | 208 +--------------------
17 files changed, 811 insertions(+), 456 deletions(-)
create mode 100644 test/test_secscan.py
create mode 100644 util/secscan/analyzer.py
create mode 100644 util/secscan/validator.py
diff --git a/app.py b/app.py
index d31c49397..1a7375000 100644
--- a/app.py
+++ b/app.py
@@ -188,8 +188,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf
reporter=MetricQueueReporter(metric_queue))
notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf)
-
-secscan_api = SecurityScannerAPI(app, config_provider)
+secscan_api = SecurityScannerAPI(app.config, config_provider, storage)
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)
diff --git a/data/database.py b/data/database.py
index 84f8b63db..e7f090de2 100644
--- a/data/database.py
+++ b/data/database.py
@@ -113,6 +113,9 @@ class CloseForLongOperation(object):
self.config_object = config_object
def __enter__(self):
+ if self.config_object.get('TESTING') == True:
+ return
+
close_db_filter(None)
def __exit__(self, type, value, traceback):
diff --git a/endpoints/api/secscan.py b/endpoints/api/secscan.py
index 146227a76..47197295f 100644
--- a/endpoints/api/secscan.py
+++ b/endpoints/api/secscan.py
@@ -2,14 +2,13 @@
import logging
import features
-import json
-import requests
from app import secscan_api
from data import model
from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
RepositoryParamResource, resource, nickname, show_if, parse_args,
query_param, truthy_bool)
+from util.secscan.api import APIRequestFailure
logger = logging.getLogger(__name__)
@@ -22,30 +21,6 @@ class SCAN_STATUS(object):
QUEUED = 'queued'
-def _call_security_api(relative_url, *args, **kwargs):
- """ Issues an HTTP call to the sec API at the given relative URL. """
- try:
- response = secscan_api.call(relative_url, None, *args, **kwargs)
- except requests.exceptions.Timeout:
- raise DownstreamIssue(payload=dict(message='API call timed out'))
- except requests.exceptions.ConnectionError:
- raise DownstreamIssue(payload=dict(message='Could not connect to downstream service'))
-
- if response.status_code == 404:
- raise NotFound()
-
- try:
- response_data = json.loads(response.text)
- except ValueError:
- raise DownstreamIssue(payload=dict(message='Non-json response from downstream service'))
-
- if response.status_code / 100 != 2:
- logger.warning('Got %s status code to call: %s', response.status_code, response.text)
- raise DownstreamIssue(payload=dict(message=response_data['Message']))
-
- return response_data
-
-
def _get_status(repo_image):
if repo_image.security_indexed_engine is not None and repo_image.security_indexed_engine >= 0:
return SCAN_STATUS.SCANNED if repo_image.security_indexed else SCAN_STATUS.FAILED
@@ -78,11 +53,16 @@ class RepositoryImageSecurity(RepositoryParamResource):
'status': _get_status(repo_image),
}
- layer_id = '%s.%s' % (repo_image.docker_image_id, repo_image.storage.uuid)
- if parsed_args.vulnerabilities:
- data = _call_security_api('layers/%s?vulnerabilities', layer_id)
- else:
- data = _call_security_api('layers/%s?features', layer_id)
+ try:
+ if parsed_args.vulnerabilities:
+ data = secscan_api.get_layer_data(repo_image, include_vulnerabilities=True)
+ else:
+ data = secscan_api.get_layer_data(repo_image, include_features=True)
+ except APIRequestFailure as arf:
+ raise DownstreamIssue(arf.message)
+
+ if data is None:
+ raise NotFound()
return {
'status': _get_status(repo_image),
diff --git a/endpoints/common.py b/endpoints/common.py
index b4c897dad..e3e37aa07 100644
--- a/endpoints/common.py
+++ b/endpoints/common.py
@@ -18,11 +18,10 @@ from app import app, oauth_apps, LoginWrappedDBUser
from auth.permissions import QuayDeferredPermissionUser
from auth import scopes
-from werkzeug.routing import BaseConverter
from functools import wraps
from config import frontend_visible_config
from external_libraries import get_external_javascript, get_external_css
-from util.secscan.api import PRIORITY_LEVELS
+from util.secscan import PRIORITY_LEVELS
from util.names import parse_namespace_repository
import features
diff --git a/endpoints/notificationevent.py b/endpoints/notificationevent.py
index b0b58f44e..925247809 100644
--- a/endpoints/notificationevent.py
+++ b/endpoints/notificationevent.py
@@ -5,7 +5,7 @@ import json
from datetime import datetime
from notificationhelper import build_event_data
from util.jinjautil import get_template_env
-from util.secscan.api import PRIORITY_LEVELS, get_priority_for_index
+from util.secscan import PRIORITY_LEVELS, get_priority_for_index
template_env = get_template_env("events")
logger = logging.getLogger(__name__)
diff --git a/initdb.py b/initdb.py
index 0e79a4d37..63cebbeb4 100644
--- a/initdb.py
+++ b/initdb.py
@@ -72,7 +72,7 @@ def __gen_image_uuid(repo, image_num):
global_image_num = count()
-def __create_subtree(repo, structure, creator_username, parent, tag_map):
+def __create_subtree(with_storage, repo, structure, creator_username, parent, tag_map):
num_nodes, subtrees, last_node_tags = structure
# create the nodes
@@ -91,7 +91,7 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
model.storage.save_torrent_info(new_image.storage, 1, 'deadbeef')
# Write some data for the storage.
- if os.environ.get('WRITE_STORAGE_FILES'):
+ if with_storage or os.environ.get('WRITE_STORAGE_FILES'):
storage_paths = StoragePaths()
paths = [storage_paths.v1_image_layer_path]
@@ -147,10 +147,10 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
found_tag.save()
for subtree in subtrees:
- __create_subtree(repo, subtree, creator_username, new_image, tag_map)
+ __create_subtree(with_storage, repo, subtree, creator_username, new_image, tag_map)
-def __generate_repository(user_obj, name, description, is_public, permissions, structure):
+def __generate_repository(with_storage, user_obj, name, description, is_public, permissions, structure):
repo = model.repository.create_repository(user_obj.username, name, user_obj)
if is_public:
@@ -165,9 +165,9 @@ def __generate_repository(user_obj, name, description, is_public, permissions, s
if isinstance(structure, list):
for leaf in structure:
- __create_subtree(repo, leaf, user_obj.username, None, {})
+ __create_subtree(with_storage, repo, leaf, user_obj.username, None, {})
else:
- __create_subtree(repo, structure, user_obj.username, None, {})
+ __create_subtree(with_storage, repo, structure, user_obj.username, None, {})
return repo
@@ -181,7 +181,7 @@ def finished_database_for_testing(testcase):
"""
testcases[testcase]['savepoint'].__exit__(True, None, None)
-def setup_database_for_testing(testcase):
+def setup_database_for_testing(testcase, with_storage=False, force_rebuild=False):
""" Called when a testcase has started using the database, indicating that
the database should be setup (if not already) and a savepoint created.
"""
@@ -190,13 +190,13 @@ def setup_database_for_testing(testcase):
if not IS_TESTING_REAL_DATABASE and not isinstance(db.obj, SqliteDatabase):
raise RuntimeError('Attempted to wipe production database!')
- if not db_initialized_for_testing.is_set():
+ if not db_initialized_for_testing.is_set() or force_rebuild:
logger.debug('Setting up DB for testing.')
# Setup the database.
wipe_database()
initialize_database()
- populate_database()
+ populate_database(with_storage=with_storage)
models_missing_data = find_models_missing_data()
if models_missing_data:
@@ -359,7 +359,7 @@ def wipe_database():
drop_model_tables(all_models, fail_silently=True)
-def populate_database(minimal=False):
+def populate_database(minimal=False, with_storage=False):
logger.debug('Populating the DB with test data.')
new_user_1 = model.user.create_user('devtable', 'password', 'jschorr@devtable.com')
@@ -428,15 +428,15 @@ def populate_database(minimal=False):
metadata=notification_metadata)
- __generate_repository(new_user_4, 'randomrepo', 'Random repo repository.', False,
+ __generate_repository(with_storage, new_user_4, 'randomrepo', 'Random repo repository.', False,
[], (4, [], ['latest', 'prod']))
- simple_repo = __generate_repository(new_user_1, 'simple', 'Simple repository.', False,
+ simple_repo = __generate_repository(with_storage, new_user_1, 'simple', 'Simple repository.', False,
[], (4, [], ['latest', 'prod']))
model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), 'local_us', {})
model.notification.create_repo_notification(simple_repo, 'repo_push', 'quay_notification', {}, {})
- __generate_repository(new_user_1, 'sharedtags',
+ __generate_repository(with_storage, new_user_1, 'sharedtags',
'Shared tags repository',
False, [(new_user_2, 'read'), (dtrobot[0], 'read')],
(2, [(3, [], ['v2.0', 'v2.1', 'v2.2']),
@@ -444,10 +444,10 @@ def populate_database(minimal=False):
['staging', '8423b58']),
(1, [], None)], None)], None))
- __generate_repository(new_user_1, 'history', 'Historical repository.', False,
+ __generate_repository(with_storage, new_user_1, 'history', 'Historical repository.', False,
[], (4, [(2, [], 'latest'), (3, [], '#latest')], None))
- __generate_repository(new_user_1, 'complex',
+ __generate_repository(with_storage, new_user_1, 'complex',
'Complex repository with many branches and tags.',
False, [(new_user_2, 'read'), (dtrobot[0], 'read')],
(2, [(3, [], 'v2.0'),
@@ -455,7 +455,7 @@ def populate_database(minimal=False):
'staging'),
(1, [], None)], None)], None))
- __generate_repository(new_user_1, 'gargantuan', None, False, [],
+ __generate_repository(with_storage, new_user_1, 'gargantuan', None, False, [],
(2, [(3, [], 'v2.0'),
(1, [(1, [(1, [], ['latest', 'prod'])],
'staging'),
@@ -465,21 +465,21 @@ def populate_database(minimal=False):
(1, [(1, [], 'v5.0'), (1, [], 'v6.0')], None)],
None))
- __generate_repository(new_user_2, 'publicrepo',
+ __generate_repository(with_storage, new_user_2, 'publicrepo',
'Public repository pullable by the world.', True,
[], (10, [], 'latest'))
- __generate_repository(outside_org, 'coolrepo',
+ __generate_repository(with_storage, outside_org, 'coolrepo',
'Some cool repo.', False,
[],
(5, [], 'latest'))
- __generate_repository(new_user_1, 'shared',
+ __generate_repository(with_storage, new_user_1, 'shared',
'Shared repository, another user can write.', False,
[(new_user_2, 'write'), (reader, 'read')],
(5, [], 'latest'))
- building = __generate_repository(new_user_1, 'building',
+ building = __generate_repository(with_storage, new_user_1, 'building',
'Empty repository which is building.',
False, [], (0, [], None))
@@ -564,10 +564,10 @@ def populate_database(minimal=False):
owners.description = 'Owners have unfetterd access across the entire org.'
owners.save()
- org_repo = __generate_repository(org, 'orgrepo', 'Repository owned by an org.', False,
+ org_repo = __generate_repository(with_storage, org, 'orgrepo', 'Repository owned by an org.', False,
[(outside_org, 'read')], (4, [], ['latest', 'prod']))
- __generate_repository(org, 'anotherorgrepo', 'Another repository owned by an org.', False,
+ __generate_repository(with_storage, org, 'anotherorgrepo', 'Another repository owned by an org.', False,
[], (4, [], ['latest', 'prod']))
creators = model.team.create_team('creators', org, 'creator', 'Creators of orgrepo.')
@@ -583,7 +583,7 @@ def populate_database(minimal=False):
model.team.add_user_to_team(creatorbot, creators)
model.team.add_user_to_team(creatoruser, creators)
- __generate_repository(new_user_1, 'superwide', None, False, [],
+ __generate_repository(with_storage, new_user_1, 'superwide', None, False, [],
[(10, [], 'latest2'),
(2, [], 'latest3'),
(2, [(1, [], 'latest11'), (2, [], 'latest12')],
diff --git a/requirements-nover.txt b/requirements-nover.txt
index 487a8edd4..c671f52b6 100644
--- a/requirements-nover.txt
+++ b/requirements-nover.txt
@@ -60,4 +60,5 @@ bintrees
redlock
semantic-version
bencode
-cryptography==1.1.2 # Remove version when https://github.com/pyca/cryptography/issues/2690 fixed
\ No newline at end of file
+cryptography==1.1.2 # Remove version when https://github.com/pyca/cryptography/issues/2690 fixed
+httmock
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index c2d5b216d..4feffb10d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -32,6 +32,7 @@ greenlet==0.4.9
gunicorn==18.0
hiredis==0.2.0
html5lib==0.9999999
+httmock==1.2.4
idna==2.0
ipaddress==1.0.16
iso8601==0.1.11
diff --git a/test/test_api_security.py b/test/test_api_security.py
index 8df483c69..c785aaee4 100644
--- a/test/test_api_security.py
+++ b/test/test_api_security.py
@@ -49,7 +49,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
SuperUserSendRecoveryEmail, ChangeLog,
SuperUserOrganizationManagement, SuperUserOrganizationList,
SuperUserAggregateLogs)
-from endpoints.api.secscan import RepositoryImagePackages, RepositoryImageVulnerabilities
+from endpoints.api.secscan import RepositoryImageSecurity
try:
@@ -4170,28 +4170,10 @@ class TestOrganizationInvoiceField(ApiTestCase):
self._run_test('DELETE', 201, 'devtable', None)
-class TestRepositoryImageVulnerabilities(ApiTestCase):
+class TestRepositoryImageSecurity(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)
- self._set_url(RepositoryImageVulnerabilities, repository='devtable/simple', imageid='fake')
-
- def test_get_anonymous(self):
- self._run_test('GET', 401, None, None)
-
- def test_get_freshuser(self):
- self._run_test('GET', 403, 'freshuser', None)
-
- def test_get_reader(self):
- self._run_test('GET', 403, 'reader', None)
-
- def test_get_devtable(self):
- self._run_test('GET', 404, 'devtable', None)
-
-
-class TestRepositoryImagePackages(ApiTestCase):
- def setUp(self):
- ApiTestCase.setUp(self)
- self._set_url(RepositoryImagePackages, repository='devtable/simple', imageid='fake')
+ self._set_url(RepositoryImageSecurity, repository='devtable/simple', imageid='fake')
def test_get_anonymous(self):
self._run_test('GET', 401, None, None)
diff --git a/test/test_api_usage.py b/test/test_api_usage.py
index 0269e3747..c46a2f8ee 100644
--- a/test/test_api_usage.py
+++ b/test/test_api_usage.py
@@ -11,6 +11,7 @@ from urllib import urlencode
from urlparse import urlparse, urlunparse, parse_qs
from playhouse.test_utils import assert_query_count, _QueryLogHandler
+from httmock import urlmatch, HTTMock
from endpoints.api import api_bp, api
from endpoints.building import PreparedBuild
@@ -52,6 +53,7 @@ from endpoints.api.repository import RepositoryList, RepositoryVisibility, Repos
from endpoints.api.permission import (RepositoryUserPermission, RepositoryTeamPermission,
RepositoryTeamPermissionList, RepositoryUserPermissionList)
from endpoints.api.superuser import SuperUserLogs, SuperUserList, SuperUserManagement
+from endpoints.api.secscan import RepositoryImageSecurity
from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile,
SuperUserCreateInitialSuperUser)
@@ -3430,6 +3432,75 @@ class TestSuperUserConfig(ApiTestCase):
self.assertTrue(json['exists'])
+
+@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
+def get_layer_success_mock(url, request):
+ vulnerabilities = [
+ {
+ "Name": "CVE-2014-9471",
+ "Namespace": "debian:8",
+ "Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
+ "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
+ "Severity": "Low",
+ "FixedBy": "9.23-5"
+ }
+ ]
+
+ features = [
+ {
+ "Name": "coreutils",
+ "Namespace": "debian:8",
+ "Version": "8.23-4",
+ "Vulnerabilities": vulnerabilities,
+ }
+ ]
+
+ if not request.url.endswith('?vulnerabilities'):
+ vulnerabilities = []
+
+ if not request.url.endswith('?features'):
+ features = []
+
+ return py_json.dumps({
+ "Layer": {
+ "Name": "17675ec01494d651e1ccf81dc9cf63959ebfeed4f978fddb1666b6ead008ed52",
+ "Namespace": "debian:8",
+ "ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
+ "IndexedByVersion": 1,
+ "Features": features
+ }
+ })
+
+
+
+class TestRepositoryImageSecurity(ApiTestCase):
+ def test_get_vulnerabilities(self):
+ self.login(ADMIN_ACCESS_USER)
+
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, 'simple', 'latest')
+
+ # Grab the security info for the tag. It should be queued.
+ response = self.getJsonResponse(RepositoryImageSecurity,
+ params=dict(repository=ADMIN_ACCESS_USER + '/simple',
+ imageid=layer.docker_image_id,
+ vulnerabilities='true'))
+ self.assertEquals('queued', response['status'])
+
+ # Mark the layer as indexed.
+ layer.security_indexed = True
+ layer.security_indexed_engine = app.config['SECURITY_SCANNER']['ENGINE_VERSION_TARGET']
+ layer.save()
+
+ # Grab the security info again.
+ with HTTMock(get_layer_success_mock):
+ response = self.getJsonResponse(RepositoryImageSecurity,
+ params=dict(repository=ADMIN_ACCESS_USER + '/simple',
+ imageid=layer.docker_image_id,
+ vulnerabilities='true'))
+ self.assertEquals('scanned', response['status'])
+ self.assertEquals(1, response['data']['Layer']['IndexedByVersion'])
+
+
class TestSuperUserManagement(ApiTestCase):
def test_get_user(self):
self.login(ADMIN_ACCESS_USER)
diff --git a/test/test_secscan.py b/test/test_secscan.py
new file mode 100644
index 000000000..9afc9ee21
--- /dev/null
+++ b/test/test_secscan.py
@@ -0,0 +1,242 @@
+import unittest
+import json
+from httmock import urlmatch, all_requests, HTTMock
+
+from app import app, config_provider, storage, notification_queue
+from initdb import setup_database_for_testing, finished_database_for_testing
+from util.secscan.api import SecurityScannerAPI, AnalyzeLayerException
+from util.secscan.analyzer import LayerAnalyzer
+from data import model
+
+
+ADMIN_ACCESS_USER = 'devtable'
+SIMPLE_REPO = 'simple'
+
+_PORT_NUMBER = 5001
+
+@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
+def get_layer_failure_mock(url, request):
+ return {'status_code': 404, 'content': json.dumps({'Error': {'Message': 'Unknown layer'}})}
+
+
+@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers$')
+def analyze_layer_badrequest_mock(url, request):
+ return {'status_code': 400, 'content': json.dumps({'Error': {'Message': 'Bad request'}})}
+
+
+@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers$')
+def analyze_layer_internal_mock(url, request):
+ return {'status_code': 500, 'content': json.dumps({'Error': {'Message': 'Internal server error'}})}
+
+
+@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers$')
+def analyze_layer_failure_mock(url, request):
+ return {'status_code': 422, 'content': json.dumps({'Error': {'Message': 'Bad layer'}})}
+
+
+@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers$')
+def analyze_layer_success_mock(url, request):
+ return {'status_code': 201, 'content': json.dumps({
+ "Layer": {
+ "Name": "523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6",
+ "Path": "/mnt/layers/523ef1d23f222195488575f52a39c729c76a8c5630c9a194139cb246fb212da6/layer.tar",
+ "ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
+ "Format": "Docker",
+ "IndexedByVersion": 1
+ }
+ })}
+
+
+@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
+def get_layer_success_mock(url, request):
+ vulnerabilities = [
+ {
+ "Name": "CVE-2014-9471",
+ "Namespace": "debian:8",
+ "Description": "The parse_datetime function in GNU coreutils allows remote attackers to cause a denial of service (crash) or possibly execute arbitrary code via a crafted date string, as demonstrated by the \"--date=TZ=\"123\"345\" @1\" string to the touch or date command.",
+ "Link": "https://security-tracker.debian.org/tracker/CVE-2014-9471",
+ "Severity": "Low",
+ "FixedBy": "9.23-5"
+ }
+ ]
+
+ features = [
+ {
+ "Name": "coreutils",
+ "Namespace": "debian:8",
+ "Version": "8.23-4",
+ "Vulnerabilities": vulnerabilities,
+ }
+ ]
+
+ if not request.url.endswith('?vulnerabilities'):
+ vulnerabilities = []
+
+ if not request.url.endswith('?features'):
+ features = []
+
+ return json.dumps({
+ "Layer": {
+ "Name": "17675ec01494d651e1ccf81dc9cf63959ebfeed4f978fddb1666b6ead008ed52",
+ "Namespace": "debian:8",
+ "ParentName": "140f9bdfeb9784cf8730e9dab5dd12fbd704151cf555ac8cae650451794e5ac2",
+ "IndexedByVersion": 1,
+ "Features": features
+ }
+ })
+
+
+@all_requests
+def response_content(url, request):
+ return {'status_code': 500, 'content': 'Unknown endpoint'}
+
+
+class TestSecurityScanner(unittest.TestCase):
+ def setUp(self):
+ # Enable direct download in fake storage.
+ storage.put_content(['local_us'], 'supports_direct_download', 'true')
+
+ # Setup the database with fake storage.
+ setup_database_for_testing(self, with_storage=True, force_rebuild=True)
+ self.app = app.test_client()
+ self.ctx = app.test_request_context()
+ self.ctx.__enter__()
+
+ self.api = SecurityScannerAPI(app.config, config_provider, storage)
+
+ def tearDown(self):
+ storage.put_content(['local_us'], 'supports_direct_download', 'false')
+ finished_database_for_testing(self)
+ self.ctx.__exit__(True, None, None)
+
+ def assertAnalyzed(self, layer, isAnalyzed, engineVersion):
+ self.assertEquals(isAnalyzed, layer.security_indexed)
+ self.assertEquals(engineVersion, layer.security_indexed_engine)
+
+ # Ensure all parent layers are marked as analyzed.
+ parents = model.image.get_parent_images(ADMIN_ACCESS_USER, SIMPLE_REPO, layer)
+ for parent in parents:
+ self.assertEquals(isAnalyzed, parent.security_indexed)
+ self.assertEquals(engineVersion, parent.security_indexed_engine)
+
+
+ def test_get_layer_success(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ with HTTMock(get_layer_success_mock, response_content):
+ result = self.api.get_layer_data(layer, include_vulnerabilities=True)
+ self.assertIsNotNone(result)
+ self.assertEquals(result['Layer']['Name'], '17675ec01494d651e1ccf81dc9cf63959ebfeed4f978fddb1666b6ead008ed52')
+
+
+ def test_get_layer_failure(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ with HTTMock(get_layer_failure_mock, response_content):
+ result = self.api.get_layer_data(layer, include_vulnerabilities=True)
+ self.assertIsNone(result)
+
+
+ def test_analyze_layer_success(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertFalse(layer.security_indexed)
+ self.assertEquals(-1, layer.security_indexed_engine)
+
+ with HTTMock(analyze_layer_success_mock, get_layer_success_mock, response_content):
+ analyzer = LayerAnalyzer(app.config, self.api)
+ analyzer.analyze_recursively(layer)
+
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertAnalyzed(layer, True, 1)
+
+
+ def test_analyze_layer_failure(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertFalse(layer.security_indexed)
+ self.assertEquals(-1, layer.security_indexed_engine)
+
+ with HTTMock(analyze_layer_failure_mock, response_content):
+ analyzer = LayerAnalyzer(app.config, self.api)
+ analyzer.analyze_recursively(layer)
+
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertAnalyzed(layer, False, 1)
+
+
+ def test_analyze_layer_internal_error(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertFalse(layer.security_indexed)
+ self.assertEquals(-1, layer.security_indexed_engine)
+
+ with HTTMock(analyze_layer_internal_mock, response_content):
+ analyzer = LayerAnalyzer(app.config, self.api)
+ analyzer.analyze_recursively(layer)
+
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertAnalyzed(layer, False, -1)
+
+
+ def test_analyze_layer_bad_request(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertFalse(layer.security_indexed)
+ self.assertEquals(-1, layer.security_indexed_engine)
+
+ with HTTMock(analyze_layer_badrequest_mock, response_content):
+ analyzer = LayerAnalyzer(app.config, self.api)
+ try:
+ analyzer.analyze_recursively(layer)
+ except AnalyzeLayerException:
+ return
+
+ self.fail('Expected exception on bad request')
+
+
+ def test_analyze_layer_missing_storage(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertFalse(layer.security_indexed)
+ self.assertEquals(-1, layer.security_indexed_engine)
+
+ # Delete the storage for the layer.
+ path = model.storage.get_layer_path(layer.storage)
+ locations = app.config['DISTRIBUTED_STORAGE_PREFERENCE']
+ storage.remove(locations, path)
+
+ with HTTMock(analyze_layer_success_mock, response_content):
+ analyzer = LayerAnalyzer(app.config, self.api)
+ analyzer.analyze_recursively(layer)
+
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertEquals(False, layer.security_indexed)
+ self.assertEquals(1, layer.security_indexed_engine)
+
+
+ def test_analyze_layer_success_events(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertFalse(layer.security_indexed)
+ self.assertEquals(-1, layer.security_indexed_engine)
+
+ # Ensure there are no existing events.
+ self.assertIsNone(notification_queue.get())
+
+ # Add a repo event for the layer.
+ repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
+ model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
+
+ with HTTMock(analyze_layer_success_mock, get_layer_success_mock, response_content):
+ analyzer = LayerAnalyzer(app.config, self.api)
+ analyzer.analyze_recursively(layer)
+
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ self.assertAnalyzed(layer, True, 1)
+
+ # Ensure an event was written for the tag.
+ queue_item = notification_queue.get()
+ self.assertIsNotNone(queue_item)
+
+ body = json.loads(queue_item.body)
+ self.assertEquals(['latest', 'prod'], body['event_data']['tags'])
+ self.assertEquals('CVE-2014-9471', body['event_data']['vulnerability']['id'])
+ self.assertEquals('Low', body['event_data']['vulnerability']['priority'])
+ self.assertTrue(body['event_data']['vulnerability']['has_fix'])
+
+
+if __name__ == '__main__':
+ unittest.main()
\ No newline at end of file
diff --git a/test/testconfig.py b/test/testconfig.py
index 0146a0988..994d3d8c4 100644
--- a/test/testconfig.py
+++ b/test/testconfig.py
@@ -26,7 +26,7 @@ class TestConfig(DefaultConfig):
DB_URI = os.environ.get('TEST_DATABASE_URI', 'sqlite:///{0}'.format(TEST_DB_FILE.name))
DB_CONNECTION_ARGS = {
'threadlocals': True,
- 'autorollback': True
+ 'autorollback': True,
}
@staticmethod
@@ -59,7 +59,8 @@ class TestConfig(DefaultConfig):
FEATURE_SECURITY_SCANNER = True
SECURITY_SCANNER = {
- 'ENDPOINT': 'http://localhost/some/invalid/path',
+ 'ENDPOINT': 'http://mockclairservice/',
+ 'API_VERSION': 'v1',
'ENGINE_VERSION_TARGET': 1,
'API_CALL_TIMEOUT': 1
}
\ No newline at end of file
diff --git a/util/secscan/__init__.py b/util/secscan/__init__.py
index e69de29bb..d09cadd73 100644
--- a/util/secscan/__init__.py
+++ b/util/secscan/__init__.py
@@ -0,0 +1,90 @@
+# NOTE: This objects are used directly in the external-notification-data and vulnerability-service
+# on the frontend, so be careful with changing their existing keys.
+PRIORITY_LEVELS = {
+ 'Unknown': {
+ 'title': 'Unknown',
+ 'index': '6',
+ 'level': 'info',
+
+ 'description': 'Unknown is either a security problem that has not been assigned ' +
+ 'to a priority yet or a priority that our system did not recognize',
+ 'banner_required': False
+ },
+
+ 'Negligible': {
+ 'title': 'Negligible',
+ 'index': '5',
+ 'level': 'info',
+
+ 'description': 'Negligible is technically a security problem, but is only theoretical ' +
+ 'in nature, requires a very special situation, has almost no install base, ' +
+ 'or does no real damage.',
+ 'banner_required': False
+ },
+
+ 'Low': {
+ 'title': 'Low',
+ 'index': '4',
+ 'level': 'warning',
+
+ 'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
+ 'requires a user-assisted attack, a small install base, or does very ' +
+ 'little damage.',
+ 'banner_required': False
+ },
+
+ 'Medium': {
+ 'title': 'Medium',
+ 'value': 'Medium',
+ 'index': '3',
+ 'level': 'warning',
+
+ 'description': 'Medium is a real security problem, and is exploitable for many people. ' +
+ 'Includes network daemon denial of service attacks, cross-site scripting, ' +
+ 'and gaining user privileges.',
+ 'banner_required': False
+ },
+
+ 'High': {
+ 'title': 'High',
+ 'value': 'High',
+ 'index': '2',
+ 'level': 'warning',
+
+ 'description': 'High is a real problem, exploitable for many people in a default installation. ' +
+ 'Includes serious remote denial of services, local root privilege escalations, ' +
+ 'or data loss.',
+ 'banner_required': False
+ },
+
+ 'Critical': {
+ 'title': 'Critical',
+ 'value': 'Critical',
+ 'index': '1',
+ 'level': 'error',
+
+ 'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
+ 'a installation of the package. Includes remote root privilege escalations, ' +
+ 'or massive data loss.',
+ 'banner_required': True
+ },
+
+ 'Defcon1': {
+ 'title': 'Defcon 1',
+ 'value': 'Defcon1',
+ 'index': '0',
+ 'level': 'error',
+
+ 'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +
+ 'by the Quay team. It requires immediate attention.',
+ 'banner_required': True
+ }
+}
+
+
+def get_priority_for_index(index):
+ for priority in PRIORITY_LEVELS:
+ if PRIORITY_LEVELS[priority]['index'] == index:
+ return priority
+
+ return 'Unknown'
\ No newline at end of file
diff --git a/util/secscan/analyzer.py b/util/secscan/analyzer.py
new file mode 100644
index 000000000..4533e933e
--- /dev/null
+++ b/util/secscan/analyzer.py
@@ -0,0 +1,138 @@
+import logging
+import logging.config
+
+from collections import defaultdict
+
+from endpoints.notificationhelper import spawn_notification
+from data.database import Image, ExternalNotificationEvent
+from data.model.tag import filter_tags_have_repository_event, get_tags_for_image
+from data.model.image import set_secscan_status, get_image_with_storage_and_parent_base
+from util.secscan.api import APIRequestFailure
+
+logger = logging.getLogger(__name__)
+
+
+class LayerAnalyzer(object):
+ """ Helper class to perform analysis of a layer via the security scanner. """
+ def __init__(self, config, api):
+ secscan_config = config.get('SECURITY_SCANNER')
+
+ self._api = api
+ self._target_version = secscan_config['ENGINE_VERSION_TARGET']
+
+
+ def analyze_recursively(self, layer):
+ """ Analyzes a layer and all its parents.
+
+ Return a tuple of two bools:
+ - The first one tells us if the layer and its parents analyzed successfully.
+ - The second one is set to False when another call pre-empted the candidate's analysis
+ for us.
+ """
+ if layer.parent_id and layer.parent.security_indexed_engine < self._target_version:
+ # The image has a parent that is not analyzed yet with this engine.
+ # Get the parent to get it's own parent and recurse.
+ try:
+ base_query = get_image_with_storage_and_parent_base()
+ parent_layer = base_query.where(Image.id == layer.parent_id).get()
+ except Image.DoesNotExist:
+ logger.warning("Image %s has Image %s as parent but doesn't exist.", layer.id,
+ layer.parent_id)
+
+ return False, set_secscan_status(layer, False, self._target_version)
+
+ cont, _ = self.analyze_recursively(parent_layer)
+ if not cont:
+ # The analysis failed for some reason and did not mark the layer as failed,
+ # thus we should not try to analyze the children of that layer.
+ # Interrupt the recursive analysis and return as no-one pre-empted us.
+ return False, True
+
+ # Now we know all parents are analyzed.
+ return self._analyze(layer)
+
+
+ def _analyze(self, layer):
+ """ Analyzes a single layer.
+
+ Return a tuple of two bools:
+ - The first one tells us if we should evaluate its children.
+ - The second one is set to False when another worker pre-empted the candidate's analysis
+ for us.
+ """
+
+ # If the parent couldn't be analyzed with the target version or higher, we can't analyze
+ # this image. Mark it as failed with the current target version.
+ if (layer.parent_id and not layer.parent.security_indexed and
+ layer.parent.security_indexed_engine >= self._target_version):
+ return True, set_secscan_status(layer, False, self._target_version)
+
+ # Analyze the image.
+ logger.info('Analyzing layer %s', layer.docker_image_id)
+ (analyzed_version, should_requeue) = self._api.analyze_layer(layer)
+
+ # If analysis failed, then determine whether we need to requeue.
+ if not analyzed_version:
+ if should_requeue:
+ # If the layer needs to be requeued, return that the children cannot be analyzed (at this
+ # time) and there was no collision with another worker.
+ return False, False
+ else:
+ # If the layer cannot be requeued, we allow the children to be analyzed, because the code
+ # path above will mark them as not analyzable, and we mark the image itself as not being
+ # analyzable.
+ return True, set_secscan_status(layer, False, self._target_version)
+
+ # Mark the image as analyzed.
+ logger.info('Analyzed layer %s successfully with version %s', layer.docker_image_id,
+ analyzed_version)
+ set_status = set_secscan_status(layer, True, analyzed_version)
+
+ # If we are the one who've done the job successfully first, get the vulnerabilities and
+ # send notifications to the repos that have a tag on that layer.
+ if set_status:
+ # Get the tags of the layer we analyzed.
+ repository_map = defaultdict(list)
+ event = ExternalNotificationEvent.get(name='vulnerability_found')
+ matching = list(filter_tags_have_repository_event(get_tags_for_image(layer.id), event))
+
+ for tag in matching:
+ repository_map[tag.repository_id].append(tag)
+
+ # If there is at least one tag,
+ # Lookup the vulnerabilities for the image, now that it is analyzed.
+ if len(repository_map) > 0:
+ logger.debug('Loading data for layer %s', layer.id)
+ try:
+ layer_data = self._api.get_layer_data(layer, include_vulnerabilities=True)
+ except APIRequestFailure:
+ layer_data = None
+
+ if layer_data is not None:
+ # Dispatch events for any detected vulnerabilities
+ logger.debug('Got data for layer %s: %s', layer.id, layer_data)
+ found_features = layer_data['Layer']['Features']
+ for repository_id in repository_map:
+ tags = repository_map[repository_id]
+
+ for feature in found_features:
+ if 'Vulnerabilities' not in feature:
+ continue
+
+ for vulnerability in feature['Vulnerabilities']:
+ event_data = {
+ 'tags': [tag.name for tag in tags],
+ 'vulnerability': {
+ 'id': vulnerability['Name'],
+ 'description': vulnerability.get('Description', None),
+ 'link': vulnerability.get('Link', None),
+ 'has_fix': 'FixedBy' in vulnerability,
+
+ # TODO: Change this key name if/when we change the event format.
+ 'priority': vulnerability.get('Severity', 'Unknown'),
+ },
+ }
+
+ spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
+
+ return True, set_status
diff --git a/util/secscan/api.py b/util/secscan/api.py
index 5041ec8ff..fd4d369fe 100644
--- a/util/secscan/api.py
+++ b/util/secscan/api.py
@@ -1,205 +1,180 @@
-import features
import logging
import requests
from data.database import CloseForLongOperation
+from data import model
+from data.model.storage import get_storage_locations
+
from urlparse import urljoin
+from util.secscan.validator import SecurityConfigValidator
logger = logging.getLogger(__name__)
-# NOTE: This objects are used directly in the external-notification-data and vulnerability-service
-# on the frontend, so be careful with changing their existing keys.
-PRIORITY_LEVELS = {
- 'Unknown': {
- 'title': 'Unknown',
- 'index': '6',
- 'level': 'info',
+class AnalyzeLayerException(Exception):
+ """ Exception raised when a layer fails to analyze due to a *client-side* issue. """
- 'description': 'Unknown is either a security problem that has not been assigned ' +
- 'to a priority yet or a priority that our system did not recognize',
- 'banner_required': False
- },
-
- 'Negligible': {
- 'title': 'Negligible',
- 'index': '5',
- 'level': 'info',
-
- 'description': 'Negligible is technically a security problem, but is only theoretical ' +
- 'in nature, requires a very special situation, has almost no install base, ' +
- 'or does no real damage.',
- 'banner_required': False
- },
-
- 'Low': {
- 'title': 'Low',
- 'index': '4',
- 'level': 'warning',
-
- 'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
- 'requires a user-assisted attack, a small install base, or does very ' +
- 'little damage.',
- 'banner_required': False
- },
-
- 'Medium': {
- 'title': 'Medium',
- 'value': 'Medium',
- 'index': '3',
- 'level': 'warning',
-
- 'description': 'Medium is a real security problem, and is exploitable for many people. ' +
- 'Includes network daemon denial of service attacks, cross-site scripting, ' +
- 'and gaining user privileges.',
- 'banner_required': False
- },
-
- 'High': {
- 'title': 'High',
- 'value': 'High',
- 'index': '2',
- 'level': 'warning',
-
- 'description': 'High is a real problem, exploitable for many people in a default installation. ' +
- 'Includes serious remote denial of services, local root privilege escalations, ' +
- 'or data loss.',
- 'banner_required': False
- },
-
- 'Critical': {
- 'title': 'Critical',
- 'value': 'Critical',
- 'index': '1',
- 'level': 'error',
-
- 'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
- 'a installation of the package. Includes remote root privilege escalations, ' +
- 'or massive data loss.',
- 'banner_required': True
- },
-
- 'Defcon1': {
- 'title': 'Defcon 1',
- 'value': 'Defcon1',
- 'index': '0',
- 'level': 'error',
-
- 'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +
- 'by the Quay team. It requires immediate attention.',
- 'banner_required': True
- }
-}
+class APIRequestFailure(Exception):
+ """ Exception raised when there is a failure to conduct an API request. """
-def get_priority_for_index(index):
- for priority in PRIORITY_LEVELS:
- if PRIORITY_LEVELS[priority]['index'] == index:
- return priority
-
- return 'Unknown'
-
-class SecurityConfigValidator(object):
- def __init__(self, app, config_provider):
- self._config_provider = config_provider
-
- if not features.SECURITY_SCANNER:
- return
-
- self._security_config = app.config['SECURITY_SCANNER']
- if self._security_config is None:
- return
-
- self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False
- self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME')
- self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME')
-
- if self._public_key and self._private_key:
- self._keys = (self._public_key, self._private_key)
- else:
- self._keys = None
-
- def _get_filepath(self, key):
- config = self._security_config
-
- if key in config:
- with self._config_provider.get_volume_file(config[key]) as f:
- return f.name
-
- return None
-
- def cert(self):
- return self._certificate
-
- def keypair(self):
- return self._keys
-
- def valid(self):
- if not features.SECURITY_SCANNER:
- return False
-
- if not self._security_config:
- logger.debug('Missing SECURITY_SCANNER block in configuration')
- return False
-
- if not 'ENDPOINT' in self._security_config:
- logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration')
- return False
-
- endpoint = self._security_config['ENDPOINT'] or ''
- if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
- logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https')
- return False
-
- if endpoint.startswith('https://') and (self._certificate is False or self._keys is None):
- logger.debug('Certificate and key pair required for talking to security worker over HTTPS')
- return False
-
- return True
+_API_METHOD_INSERT = 'layers'
+_API_METHOD_GET_LAYER = 'layers/%s'
+_API_METHOD_GET_WITH_VULNERABILITIES_FLAG = '?vulnerabilities'
+_API_METHOD_GET_WITH_FEATURES_FLAG = '?features'
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (Clair). """
- def __init__(self, app, config_provider):
- self.app = app
+ def __init__(self, config, config_provider, storage):
+ self.config = config
self.config_provider = config_provider
+
+ self._storage = storage
self._security_config = None
- config_validator = SecurityConfigValidator(app, config_provider)
+ config_validator = SecurityConfigValidator(config, config_provider)
if not config_validator.valid():
logger.warning('Invalid config provided to SecurityScannerAPI')
return
- self._security_config = app.config.get('SECURITY_SCANNER')
+ self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
+
+ self._security_config = config.get('SECURITY_SCANNER')
+ self._target_version = self._security_config['ENGINE_VERSION_TARGET']
+
self._certificate = config_validator.cert()
self._keys = config_validator.keypair()
- def check_layer_vulnerable(self, layer_id, cve_id):
- """ Checks with Clair whether the given layer is vulnerable to the given CVE. """
- try:
- body = {
- 'LayersIDs': [layer_id]
+
+ def _get_image_url(self, image):
+ """ Gets the download URL for an image and if the storage doesn't exist,
+ returns None.
+ """
+ path = model.storage.get_layer_path(image.storage)
+ locations = self._default_storage_locations
+
+ if not self._storage.exists(locations, path):
+ locations = get_storage_locations(image.storage.uuid)
+
+ if not locations or not self._storage.exists(locations, path):
+ logger.warning('Could not find a valid location to download layer %s.%s out of %s',
+ image.docker_image_id, image.storage.uuid, locations)
+ return None
+
+ uri = self._storage.get_direct_download_url(locations, path)
+ if uri is None:
+ # Handle local storage.
+ local_storage_enabled = False
+ for storage_type, _ in self.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
+ if storage_type == 'LocalStorage':
+ local_storage_enabled = True
+
+ if local_storage_enabled:
+ # TODO: fix to use the proper local storage path.
+ uri = path
+ else:
+ logger.warning('Could not get image URL and local storage was not enabled')
+ return None
+
+ return uri
+
+
+ def _new_analyze_request(self, image):
+ """ Create the request body to submit the given image for analysis. If the image's URL cannot
+ be found, returns None.
+ """
+ url = self._get_image_url(image)
+ if url is None:
+ return None
+
+ request = {
+ 'Layer': {
+ 'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid),
+ 'Path': url,
+ 'Format': 'Docker'
}
- response = self.call('vulnerabilities/%s/affected-layers', body, cve_id)
- except requests.exceptions.RequestException:
- logger.exception('Got exception when trying to call Clair endpoint')
- return False
+ }
- if response.status_code != 200:
- return False
+ if image.parent.docker_image_id and image.parent.storage.uuid:
+ request['Layer']['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
+ image.parent.storage.uuid)
+ return request
+
+
+ def analyze_layer(self, layer):
+ """ Posts the given layer to the security scanner for analysis, blocking until complete.
+ Returns a tuple containing the analysis version (on success, None on failure) and
+ whether the request should be retried.
+ """
+ request = self._new_analyze_request(layer)
+ if not request:
+ return None, False
+
+ logger.info('Analyzing layer %s', request['Layer']['Name'])
try:
- response_data = response.json()
- except ValueError:
- logger.exception('Got exception when trying to parse Clair response')
- return False
+ response = self._call(_API_METHOD_INSERT, request)
+ json_response = response.json()
+ except requests.exceptions.Timeout:
+ logger.exception('Timeout when trying to post layer data response for %s', layer.id)
+ return None, True
+ except requests.exceptions.ConnectionError:
+ logger.exception('Connection error when trying to post layer data response for %s', layer.id)
+ return None, True
+ except (requests.exceptions.RequestException, ValueError):
+ logger.exception('Failed to post layer data response for %s', layer.id)
+ return None, False
- if (not layer_id in response_data or
- not response_data[layer_id].get('Vulnerable', False)):
- return False
+ # Handle any errors from the security scanner.
+ if response.status_code != 201:
+ message = json_response.get('Error').get('Message', '')
+ logger.warning('A warning event occurred when analyzing layer %s (status code %s): %s',
+ request['Layer']['Name'], response.status_code, message)
- return True
+ # 400 means the layer could not be analyzed due to a bad request.
+ if response.status_code == 400:
+ logger.error('Bad request when calling security scanner for layer %s: %s',
+ response.status_code, json_response)
+ raise AnalyzeLayerException('Bad request to security scanner')
- def call(self, relative_url, body=None, *args, **kwargs):
+ # 422 means that the layer could not be analyzed:
+ # - the layer could not be extracted (manifest?)
+ # - the layer operating system / package manager is unsupported
+ return None, response.status_code != 422
+
+ api_version = json_response['Layer']['IndexedByVersion']
+ return api_version, False
+
+
+ def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
+ """ Returns the layer data for the specified layer. On error, returns None. """
+ layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+ try:
+ flag = ''
+ if include_features:
+ flag = _API_METHOD_GET_WITH_FEATURES_FLAG
+
+ if include_vulnerabilities:
+ flag = _API_METHOD_GET_WITH_VULNERABILITIES_FLAG
+
+ response = self._call(_API_METHOD_GET_LAYER + flag, None, layer_id)
+ logger.debug('Got response %s for vulnerabilities for layer %s',
+ response.status_code, layer_id)
+ except requests.exceptions.Timeout:
+ raise APIRequestFailure('API call timed out')
+ except requests.exceptions.ConnectionError:
+ raise APIRequestFailure('Could not connect to security service')
+ except (requests.exceptions.RequestException, ValueError):
+ logger.exception('Failed to get layer data response for %s', layer.id)
+ raise APIRequestFailure()
+
+ if response.status_code == 404:
+ return None
+
+ return response.json()
+
+
+ def _call(self, relative_url, body=None, *args, **kwargs):
""" Issues an HTTP call to the sec API at the given relative URL.
This function disconnects from the database while awaiting a response
from the API server.
@@ -211,14 +186,16 @@ class SecurityScannerAPI(object):
api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
url = urljoin(api_url, relative_url % args)
- client = self.app.config['HTTPCLIENT']
+ client = self.config['HTTPCLIENT']
timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
logger.debug('Looking up sec information: %s', url)
- with CloseForLongOperation(self.app.config):
+ with CloseForLongOperation(self.config):
if body is not None:
+ logger.debug('POSTing security URL %s', url)
return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self._keys,
verify=self._certificate)
else:
+ logger.debug('GETing security URL %s', url)
return client.get(url, params=kwargs, timeout=timeout, cert=self._keys,
verify=self._certificate)
diff --git a/util/secscan/validator.py b/util/secscan/validator.py
new file mode 100644
index 000000000..44739a825
--- /dev/null
+++ b/util/secscan/validator.py
@@ -0,0 +1,65 @@
+import features
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SecurityConfigValidator(object):
+ """ Helper class for validating the security scanner configuration. """
+ def __init__(self, config, config_provider):
+ self._config_provider = config_provider
+
+ if not features.SECURITY_SCANNER:
+ return
+
+ self._security_config = config['SECURITY_SCANNER']
+ if self._security_config is None:
+ return
+
+ self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False
+ self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME')
+ self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME')
+
+ if self._public_key and self._private_key:
+ self._keys = (self._public_key, self._private_key)
+ else:
+ self._keys = None
+
+ def _get_filepath(self, key):
+ config = self._security_config
+
+ if key in config:
+ with self._config_provider.get_volume_file(config[key]) as f:
+ return f.name
+
+ return None
+
+ def cert(self):
+ return self._certificate
+
+ def keypair(self):
+ return self._keys
+
+ def valid(self):
+ if not features.SECURITY_SCANNER:
+ return False
+
+ if not self._security_config:
+ logger.debug('Missing SECURITY_SCANNER block in configuration')
+ return False
+
+ if not 'ENDPOINT' in self._security_config:
+ logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration')
+ return False
+
+ endpoint = self._security_config['ENDPOINT'] or ''
+ if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
+ logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https')
+ return False
+
+ if endpoint.startswith('https://') and (self._certificate is False or self._keys is None):
+ logger.debug('Certificate and key pair required for talking to security worker over HTTPS')
+ return False
+
+ return True
+
diff --git a/workers/securityworker.py b/workers/securityworker.py
index 51998e068..a39bccb71 100644
--- a/workers/securityworker.py
+++ b/workers/securityworker.py
@@ -1,98 +1,37 @@
import logging
import logging.config
-import requests
import features
import time
from peewee import fn
-from collections import defaultdict
-from app import app, config_provider, storage, secscan_api
-from endpoints.notificationhelper import spawn_notification
+from app import app, config_provider, secscan_api
from workers.worker import Worker
-from data import model
-from data.database import (Image, UseThenDisconnect, ExternalNotificationEvent)
-from data.model.tag import filter_tags_have_repository_event, get_tags_for_image
-from data.model.image import set_secscan_status, get_image_with_storage_and_parent_base
-from data.model.storage import get_storage_locations
+from data.database import Image, UseThenDisconnect
+from data.model.image import get_image_with_storage_and_parent_base
from util.secscan.api import SecurityConfigValidator
+from util.secscan.analyzer import LayerAnalyzer
from util.migrate.allocator import yield_random_entries
BATCH_SIZE = 50
INDEXING_INTERVAL = 30
-API_METHOD_INSERT = '/v1/layers'
-API_METHOD_GET_WITH_VULNERABILITIES = '/v1/layers/%s?vulnerabilities'
logger = logging.getLogger(__name__)
class SecurityWorker(Worker):
def __init__(self):
super(SecurityWorker, self).__init__()
- validator = SecurityConfigValidator(app, config_provider)
+ validator = SecurityConfigValidator(app.config, config_provider)
if validator.valid():
secscan_config = app.config.get('SECURITY_SCANNER')
- self._api = secscan_config['ENDPOINT']
self._target_version = secscan_config['ENGINE_VERSION_TARGET']
- self._default_storage_locations = app.config['DISTRIBUTED_STORAGE_PREFERENCE']
- self._cert = validator.cert()
- self._keys = validator.keypair()
+ self._analyzer = LayerAnalyzer(app.config, secscan_api)
self.add_operation(self._index_images, INDEXING_INTERVAL)
else:
logger.warning('Failed to validate security scan configuration')
- def _new_request(self, image):
- """ Create the request body to submit the given image for analysis. """
- url = self._get_image_url(image)
- if url is None:
- return None
-
- request = {
- 'Layer': {
- 'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid),
- 'Path': url,
- 'Format': 'Docker'
- }
- }
-
- if image.parent.docker_image_id and image.parent.storage.uuid:
- request['Layer']['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
- image.parent.storage.uuid)
-
- return request
-
- def _get_image_url(self, image):
- """ Gets the download URL for an image and if the storage doesn't exist,
- marks the image as unindexed. """
- path = model.storage.get_layer_path(image.storage)
- locations = self._default_storage_locations
-
- if not storage.exists(locations, path):
- locations = get_storage_locations(image.storage.uuid)
-
- if not locations or not storage.exists(locations, path):
- logger.warning('Could not find a valid location to download layer %s.%s',
- image.docker_image_id, image.storage.uuid)
- set_secscan_status(image, False, self._target_version)
- return None
-
- uri = storage.get_direct_download_url(locations, path)
- if uri is None:
- # Handle local storage
- local_storage_enabled = False
- for storage_type, _ in app.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
- if storage_type == 'LocalStorage':
- local_storage_enabled = True
-
- if local_storage_enabled:
- uri = path
- else:
- logger.warning('Could not get image URL and local storage was not enabled')
- return None
-
- return uri
-
def _index_images(self):
def batch_query():
base_query = get_image_with_storage_and_parent_base()
@@ -106,144 +45,11 @@ class SecurityWorker(Worker):
with UseThenDisconnect(app.config):
for candidate, abt in yield_random_entries(batch_query, Image.id, BATCH_SIZE, max_id, min_id):
- _, continue_batch = self._analyze_recursively(candidate)
+ _, continue_batch = self._analyzer.analyze_recursively(candidate)
if not continue_batch:
logger.info('Another worker pre-empted us for layer: %s', candidate.id)
abt.set()
- def _analyze_recursively(self, layer):
- """ Analyzes a layer and all its parents """
- if layer.parent_id and layer.parent.security_indexed_engine < self._target_version:
- # The image has a parent that is not analyzed yet with this engine.
- # Get the parent to get it's own parent and recurse.
- try:
- base_query = get_image_with_storage_and_parent_base()
- parent_layer = base_query.where(Image.id == layer.parent_id).get()
- except Image.DoesNotExist:
- logger.warning("Image %s has Image %s as parent but doesn't exist.", layer.id,
- layer.parent_id)
-
- return False, set_secscan_status(layer, False, self._target_version)
-
- cont, _ = self._analyze_recursively(parent_layer)
- if not cont:
- # The analysis failed for some reason and did not mark the layer as failed,
- # thus we should not try to analyze the children of that layer.
- # Interrupt the recursive analysis and return as no-one pre-empted us.
- return False, True
-
- # Now we know all parents are analyzed.
- return self._analyze(layer)
-
- def _analyze(self, layer):
- """ Analyzes a single layer.
- Return two bools, the first one tells us if we should evaluate its children, the second
- one is set to False when another worker pre-empted the candidate's analysis for us. """
-
- # If the parent couldn't be analyzed with the target version or higher, we can't analyze
- # this image. Mark it as failed with the current target version.
- if (layer.parent_id and not layer.parent.security_indexed and
- layer.parent.security_indexed_engine >= self._target_version):
- return True, set_secscan_status(layer, False, self._target_version)
-
- request = self._new_request(layer)
- if request is None:
- return False, True
-
- # Analyze the image.
- try:
- logger.info('Analyzing layer %s', request['Layer']['Name'])
- # Using invalid certificates doesn't return proper errors because of
- # https://github.com/shazow/urllib3/issues/556
- http_response = requests.post(self._api + API_METHOD_INSERT, json=request,
- cert=self._keys, verify=self._cert)
- json_response = http_response.json()
- except (requests.exceptions.RequestException, ValueError):
- logger.exception('An exception occurred when analyzing layer %s', request['Layer']['Name'])
- return False, True
-
- # Handle any errors from the security scanner.
- if http_response.status_code != 201:
- message = json_response.get('Error').get('Message', '')
- logger.warning('A warning event occurred when analyzing layer %s (status code %s): %s',
- request['Layer']['Name'], http_response.status_code, message)
-
- # 422 means that the layer could not be analyzed:
- # - the layer could not be extracted (manifest?)
- # - the layer operating system / package manager is unsupported
- # Set the layer as failed.
- if http_response.status_code == 422:
- return True, set_secscan_status(layer, False, self._target_version)
- else:
- return False, True
-
- # Verify that the version matches.
- api_version = json_response['Layer']['IndexedByVersion']
- if api_version < self._target_version:
- logger.warning('An engine runs on version %d but the target version is %d', api_version,
- self._target_version)
-
- # Mark the image as analyzed.
- logger.info('Analyzed layer %s successfully', request['Layer']['Name'])
- set_status = set_secscan_status(layer, True, api_version)
-
- # If we are the one who've done the job successfully first, get the vulnerabilities and
- # send notifications to the repos that have a tag on that layer.
- # TODO(josephschorr): Adapt this depending on the new notification format we adopt.
- # if set_status:
- # # Get the tags of the layer we analyzed.
- # repository_map = defaultdict(list)
- # event = ExternalNotificationEvent.get(name='vulnerability_found')
- # matching = list(filter_tags_have_repository_event(get_tags_for_image(layer.id), event))
- #
- # for tag in matching:
- # repository_map[tag.repository_id].append(tag)
- #
- # # If there is at least one tag,
- # # Lookup the vulnerabilities for the image, now that it is analyzed.
- # if len(repository_map) > 0:
- # logger.debug('Loading vulnerabilities for layer %s', layer.id)
- # sec_data = self._get_vulnerabilities(layer)
- #
- # if sec_data is not None:
- # # Dispatch events for any detected vulnerabilities
- # logger.debug('Got vulnerabilities for layer %s: %s', layer.id, sec_data)
- #
- # for repository_id in repository_map:
- # tags = repository_map[repository_id]
- #
- # for vuln in sec_data['Vulnerabilities']:
- # event_data = {
- # 'tags': [tag.name for tag in tags],
- # 'vulnerability': {
- # 'id': vuln['Name'],
- # 'description': vuln['Description'],
- # 'link': vuln['Link'],
- # 'priority': vuln['Priority'],
- # },
- # }
- #
- # spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
-
- return True, set_status
-
- def _get_vulnerabilities(self, layer):
- """ Returns the vulnerabilities detected (if any) or None on error. """
- try:
- response = secscan_api.call(self._api + API_METHOD_GET_WITH_VULNERABILITIES, None,
- '%s.%s' % (layer.docker_image_id, layer.storage.uuid))
-
- logger.debug('Got response %s for vulnerabilities for layer %s',
- response.status_code, layer.id)
-
- if response.status_code == 404:
- return None
- except (requests.exceptions.RequestException, ValueError):
- logger.exception('Failed to get vulnerability response for %s', layer.id)
- return None
-
- return response.json()
-
if __name__ == '__main__':
if not features.SECURITY_SCANNER:
From f498e92d5822b7ecb7ef426c77f04e08139371f4 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <josephschorr@users.noreply.github.com>
Date: Thu, 25 Feb 2016 15:58:42 -0500
Subject: [PATCH 06/12] Implement against new Clair paginated notification
system
---
config.py | 2 +-
data/queue.py | 12 +-
endpoints/secscan.py | 25 +--
test/test_api_usage.py | 4 +-
test/test_secscan.py | 204 +++++++++++++++++++++++-
util/secscan/__init__.py | 14 +-
util/secscan/api.py | 92 +++++++++--
util/secscan/notifier.py | 103 ++++++++++++
workers/queueworker.py | 5 +-
workers/security_notification_worker.py | 87 +++-------
10 files changed, 447 insertions(+), 101 deletions(-)
create mode 100644 util/secscan/notifier.py
diff --git a/config.py b/config.py
index b3d230138..3e0b391f2 100644
--- a/config.py
+++ b/config.py
@@ -129,7 +129,7 @@ class DefaultConfig(object):
NOTIFICATION_QUEUE_NAME = 'notification'
DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild'
REPLICATION_QUEUE_NAME = 'imagestoragereplication'
- SECSCAN_NOTIFICATION_QUEUE_NAME = 'secscan_notification'
+ SECSCAN_NOTIFICATION_QUEUE_NAME = 'security_notification'
# Super user config. Note: This MUST BE an empty list for the default config.
SUPER_USERS = []
diff --git a/data/queue.py b/data/queue.py
index 7503e5764..5b4408a4c 100644
--- a/data/queue.py
+++ b/data/queue.py
@@ -228,16 +228,26 @@ class WorkQueue(object):
except QueueItem.DoesNotExist:
return False
- def extend_processing(self, item, seconds_from_now, minimum_extension=MINIMUM_EXTENSION):
+ def extend_processing(self, item, seconds_from_now, minimum_extension=MINIMUM_EXTENSION,
+ updated_data=None):
with self._transaction_factory(db):
try:
queue_item = self._item_by_id_for_update(item.id)
new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now)
+ has_change = False
# Only actually write the new expiration to the db if it moves the expiration some minimum
if new_expiration - queue_item.processing_expires > minimum_extension:
queue_item.processing_expires = new_expiration
+ has_change = True
+
+ if updated_data is not None:
+ queue_item.body = updated_data
+ has_change = True
+
+ if has_change:
queue_item.save()
+
except QueueItem.DoesNotExist:
return
diff --git a/endpoints/secscan.py b/endpoints/secscan.py
index 4326ea621..cd842f88e 100644
--- a/endpoints/secscan.py
+++ b/endpoints/secscan.py
@@ -3,23 +3,30 @@ import json
import features
-from app import secscan_notification_queue
-from flask import request, make_response, Blueprint
+from app import secscan_notification_queue, secscan_api
+from flask import request, make_response, Blueprint, abort
from endpoints.common import route_show_if
logger = logging.getLogger(__name__)
secscan = Blueprint('secscan', __name__)
@route_show_if(features.SECURITY_SCANNER)
-@secscan.route('/notification', methods=['POST'])
+@secscan.route('/notify', methods=['POST'])
def secscan_notification():
data = request.get_json()
- logger.debug('Got notification from Clair: %s', data)
+ logger.debug('Got notification from Security Scanner: %s', data)
+ if 'Notification' not in data:
+ abort(400)
- content = data['Content']
- layer_ids = content.get('NewIntroducingLayersIDs', content.get('IntroducingLayersIDs', []))
- if not layer_ids:
- return make_response('Okay')
+ notification = data['Notification']
+
+ # Queue the notification to be processed.
+ item_id = secscan_notification_queue.put(['named', notification['Name']],
+ json.dumps(notification))
+
+ # Mark the notification as read.
+ if not secscan_api.mark_notification_read(notification['Name']):
+ secscan_notification_queue.cancel(item_id)
+ abort(400)
- secscan_notification_queue.put(['notification', data['Name']], json.dumps(data))
return make_response('Okay')
diff --git a/test/test_api_usage.py b/test/test_api_usage.py
index c46a2f8ee..cd700fc1c 100644
--- a/test/test_api_usage.py
+++ b/test/test_api_usage.py
@@ -3455,10 +3455,10 @@ def get_layer_success_mock(url, request):
}
]
- if not request.url.endswith('?vulnerabilities'):
+ if not request.url.index('vulnerabilities') > 0:
vulnerabilities = []
- if not request.url.endswith('?features'):
+ if not request.url.index('features') > 0:
features = []
return py_json.dumps({
diff --git a/test/test_secscan.py b/test/test_secscan.py
index 9afc9ee21..e4783e7a9 100644
--- a/test/test_secscan.py
+++ b/test/test_secscan.py
@@ -1,11 +1,13 @@
import unittest
import json
+import os
from httmock import urlmatch, all_requests, HTTMock
from app import app, config_provider, storage, notification_queue
from initdb import setup_database_for_testing, finished_database_for_testing
from util.secscan.api import SecurityScannerAPI, AnalyzeLayerException
from util.secscan.analyzer import LayerAnalyzer
+from util.secscan.notifier import process_notification_data
from data import model
@@ -69,10 +71,10 @@ def get_layer_success_mock(url, request):
}
]
- if not request.url.endswith('?vulnerabilities'):
+ if not request.url.find('vulnerabilities') > 0:
vulnerabilities = []
- if not request.url.endswith('?features'):
+ if not request.url.find('features') > 0:
features = []
return json.dumps({
@@ -97,7 +99,8 @@ class TestSecurityScanner(unittest.TestCase):
storage.put_content(['local_us'], 'supports_direct_download', 'true')
# Setup the database with fake storage.
- setup_database_for_testing(self, with_storage=True, force_rebuild=True)
+ force_rebuild = os.environ.get('SKIP_REBUILD') != 'true'
+ setup_database_for_testing(self, with_storage=True, force_rebuild=force_rebuild)
self.app = app.test_client()
self.ctx = app.test_request_context()
self.ctx.__enter__()
@@ -238,5 +241,200 @@ class TestSecurityScanner(unittest.TestCase):
self.assertTrue(body['event_data']['vulnerability']['has_fix'])
+ def _get_notification_data(self, new_layer_ids, old_layer_ids, new_severity='Low'):
+ return {
+ "Name": "ec45ec87-bfc8-4129-a1c3-d2b82622175a",
+ "Created": "1456247389",
+ "Notified": "1456246708",
+ "Limit": 2,
+ "New": {
+ "Vulnerability": {
+ "Name": "CVE-TEST",
+ "Namespace": "debian:8",
+ "Description": "New CVE",
+ "Severity": new_severity,
+ "FixedIn": [
+ {
+ "Name": "grep",
+ "Namespace": "debian:8",
+ "Version": "2.25"
+ }
+ ]
+ },
+ "LayersIntroducingVulnerability": new_layer_ids,
+ },
+ "Old": {
+ "Vulnerability": {
+ "Name": "CVE-TEST",
+ "Namespace": "debian:8",
+ "Description": "New CVE",
+ "Severity": "Low",
+ "FixedIn": []
+ },
+ "LayersIntroducingVulnerability": old_layer_ids,
+ }
+ }
+
+
+ def test_notification_new_layers_not_vulnerable(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+
+ # Add a repo event for the layer.
+ repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
+ model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
+
+ @urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
+ def get_matching_layer_not_vulnerable(url, request):
+ return json.dumps({
+ "Layer": {
+ "Name": layer_id,
+ "Namespace": "debian:8",
+ "IndexedByVersion": 1,
+ "Features": [
+ {
+ "Name": "coreutils",
+ "Namespace": "debian:8",
+ "Version": "8.23-4",
+ "Vulnerabilities": [], # Report not vulnerable.
+ }
+ ]
+ }
+ })
+
+ # Ensure that there are no event queue items for the layer.
+ self.assertIsNone(notification_queue.get())
+
+ # Fire off the notification processing.
+ with HTTMock(get_matching_layer_not_vulnerable, response_content):
+ notification_data = self._get_notification_data([layer_id], [])
+ self.assertTrue(process_notification_data(notification_data))
+
+ # Ensure that there are no event queue items for the layer.
+ self.assertIsNone(notification_queue.get())
+
+
+ def test_notification_new_layers(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+
+ # Add a repo event for the layer.
+ repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
+ model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
+
+ @urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
+ def get_matching_layer_vulnerable(url, request):
+ return json.dumps({
+ "Layer": {
+ "Name": layer_id,
+ "Namespace": "debian:8",
+ "IndexedByVersion": 1,
+ "Features": [
+ {
+ "Name": "coreutils",
+ "Namespace": "debian:8",
+ "Version": "8.23-4",
+ "Vulnerabilities": [
+ {
+ "Name": "CVE-TEST",
+ "Namespace": "debian:8",
+ "Severity": "Low",
+ }
+ ],
+ }
+ ]
+ }
+ })
+
+ # Ensure that there are no event queue items for the layer.
+ self.assertIsNone(notification_queue.get())
+
+ # Fire off the notification processing.
+ with HTTMock(get_matching_layer_vulnerable, response_content):
+ notification_data = self._get_notification_data([layer_id], [])
+ self.assertTrue(process_notification_data(notification_data))
+
+ # Ensure an event was written for the tag.
+ queue_item = notification_queue.get()
+ self.assertIsNotNone(queue_item)
+
+ body = json.loads(queue_item.body)
+ self.assertEquals(['prod', 'latest'], body['event_data']['tags'])
+ self.assertEquals('CVE-TEST', body['event_data']['vulnerability']['id'])
+ self.assertEquals('Low', body['event_data']['vulnerability']['priority'])
+ self.assertTrue(body['event_data']['vulnerability']['has_fix'])
+
+
+ def test_notification_no_new_layers(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+
+ # Add a repo event for the layer.
+ repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
+ model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
+
+ # Ensure that there are no event queue items for the layer.
+ self.assertIsNone(notification_queue.get())
+
+ # Fire off the notification processing.
+ with HTTMock(response_content):
+ notification_data = self._get_notification_data([layer_id], [layer_id])
+ self.assertTrue(process_notification_data(notification_data))
+
+ # Ensure that there are no event queue items for the layer.
+ self.assertIsNone(notification_queue.get())
+
+
+ def test_notification_no_new_layers_increased_severity(self):
+ layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
+ layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+
+ # Add a repo event for the layer.
+ repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
+ model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
+
+ @urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
+ def get_matching_layer_vulnerable(url, request):
+ return json.dumps({
+ "Layer": {
+ "Name": layer_id,
+ "Namespace": "debian:8",
+ "IndexedByVersion": 1,
+ "Features": [
+ {
+ "Name": "coreutils",
+ "Namespace": "debian:8",
+ "Version": "8.23-4",
+ "Vulnerabilities": [
+ {
+ "Name": "CVE-TEST",
+ "Namespace": "debian:8",
+ "Severity": "Low",
+ }
+ ],
+ }
+ ]
+ }
+ })
+
+ # Ensure that there are no event queue items for the layer.
+ self.assertIsNone(notification_queue.get())
+
+ # Fire off the notification processing.
+ with HTTMock(get_matching_layer_vulnerable, response_content):
+ notification_data = self._get_notification_data([layer_id], [layer_id], new_severity='High')
+ self.assertTrue(process_notification_data(notification_data))
+
+ # Ensure an event was written for the tag.
+ queue_item = notification_queue.get()
+ self.assertIsNotNone(queue_item)
+
+ body = json.loads(queue_item.body)
+ self.assertEquals(['prod', 'latest'], body['event_data']['tags'])
+ self.assertEquals('CVE-TEST', body['event_data']['vulnerability']['id'])
+ self.assertEquals('High', body['event_data']['vulnerability']['priority'])
+ self.assertTrue(body['event_data']['vulnerability']['has_fix'])
+
+
if __name__ == '__main__':
unittest.main()
\ No newline at end of file
diff --git a/util/secscan/__init__.py b/util/secscan/__init__.py
index d09cadd73..1588e2a16 100644
--- a/util/secscan/__init__.py
+++ b/util/secscan/__init__.py
@@ -3,7 +3,7 @@
PRIORITY_LEVELS = {
'Unknown': {
'title': 'Unknown',
- 'index': '6',
+ 'index': 6,
'level': 'info',
'description': 'Unknown is either a security problem that has not been assigned ' +
@@ -13,7 +13,7 @@ PRIORITY_LEVELS = {
'Negligible': {
'title': 'Negligible',
- 'index': '5',
+ 'index': 5,
'level': 'info',
'description': 'Negligible is technically a security problem, but is only theoretical ' +
@@ -24,7 +24,7 @@ PRIORITY_LEVELS = {
'Low': {
'title': 'Low',
- 'index': '4',
+ 'index': 4,
'level': 'warning',
'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
@@ -36,7 +36,7 @@ PRIORITY_LEVELS = {
'Medium': {
'title': 'Medium',
'value': 'Medium',
- 'index': '3',
+ 'index': 3,
'level': 'warning',
'description': 'Medium is a real security problem, and is exploitable for many people. ' +
@@ -48,7 +48,7 @@ PRIORITY_LEVELS = {
'High': {
'title': 'High',
'value': 'High',
- 'index': '2',
+ 'index': 2,
'level': 'warning',
'description': 'High is a real problem, exploitable for many people in a default installation. ' +
@@ -60,7 +60,7 @@ PRIORITY_LEVELS = {
'Critical': {
'title': 'Critical',
'value': 'Critical',
- 'index': '1',
+ 'index': 1,
'level': 'error',
'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
@@ -72,7 +72,7 @@ PRIORITY_LEVELS = {
'Defcon1': {
'title': 'Defcon 1',
'value': 'Defcon1',
- 'index': '0',
+ 'index': 0,
'level': 'error',
'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +
diff --git a/util/secscan/api.py b/util/secscan/api.py
index fd4d369fe..4fc652b06 100644
--- a/util/secscan/api.py
+++ b/util/secscan/api.py
@@ -19,8 +19,8 @@ class APIRequestFailure(Exception):
_API_METHOD_INSERT = 'layers'
_API_METHOD_GET_LAYER = 'layers/%s'
-_API_METHOD_GET_WITH_VULNERABILITIES_FLAG = '?vulnerabilities'
-_API_METHOD_GET_WITH_FEATURES_FLAG = '?features'
+_API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s'
+_API_METHOD_GET_NOTIFICATION = 'notifications/%s'
class SecurityScannerAPI(object):
@@ -113,7 +113,7 @@ class SecurityScannerAPI(object):
logger.info('Analyzing layer %s', request['Layer']['Name'])
try:
- response = self._call(_API_METHOD_INSERT, request)
+ response = self._call('POST', _API_METHOD_INSERT, request)
json_response = response.json()
except requests.exceptions.Timeout:
logger.exception('Timeout when trying to post layer data response for %s', layer.id)
@@ -146,35 +146,94 @@ class SecurityScannerAPI(object):
return api_version, False
+ def check_layer_vulnerable(self, layer_id, cve_name):
+ """ Checks to see if the layer with the given ID is vulnerable to the specified CVE. """
+ layer_data = self._get_layer_data(layer_id, include_vulnerabilities=True)
+ if layer_data is None or 'Layer' not in layer_data or 'Features' not in layer_data['Layer']:
+ return False
+
+ for feature in layer_data['Layer']['Features']:
+ for vuln in feature.get('Vulnerabilities', []):
+ if vuln['Name'] == cve_name:
+ return True
+
+ return False
+
+
+ def get_notification(self, notification_name, layer_limit=10, page=None):
+ """ Gets the data for a specific notification, with optional page token.
+ Returns a tuple of the data (None on failure) and whether to retry.
+ """
+ try:
+ params = {
+ 'limit': layer_limit
+ }
+
+ if page is not None:
+ params['page'] = page
+
+ response = self._call('GET', _API_METHOD_GET_NOTIFICATION % notification_name, params=params)
+ json_response = response.json()
+ except requests.exceptions.Timeout:
+ logger.exception('Timeout when trying to get notification for %s', notification_name)
+ return None, True
+ except requests.exceptions.ConnectionError:
+ logger.exception('Connection error when trying to get notification for %s', notification_name)
+ return None, True
+ except (requests.exceptions.RequestException, ValueError):
+ logger.exception('Failed to get notification for %s', notification_name)
+ return None, False
+
+ if response.status_code != 200:
+ return None, response.status_code != 404 and response.status_code != 400
+
+ return json_response, False
+
+
+ def mark_notification_read(self, notification_name):
+ """ Marks a security scanner notification as read. """
+ try:
+ response = self._call('DELETE', _API_METHOD_MARK_NOTIFICATION_READ % notification_name)
+ return response.status_code == 200
+ except requests.exceptions.RequestException:
+ logger.exception('Failed to mark notification as read: %s', notification_name)
+ return False
+
+
def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
""" Returns the layer data for the specified layer. On error, returns None. """
layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+ return self._get_layer_data(layer_id, include_features, include_vulnerabilities)
+
+
+ def _get_layer_data(self, layer_id, include_features=False, include_vulnerabilities=False):
try:
- flag = ''
+ params = {}
if include_features:
- flag = _API_METHOD_GET_WITH_FEATURES_FLAG
+ params = {'features': True}
if include_vulnerabilities:
- flag = _API_METHOD_GET_WITH_VULNERABILITIES_FLAG
+ params = {'vulnerabilities': True}
- response = self._call(_API_METHOD_GET_LAYER + flag, None, layer_id)
+ response = self._call('GET', _API_METHOD_GET_LAYER % layer_id, params=params)
logger.debug('Got response %s for vulnerabilities for layer %s',
response.status_code, layer_id)
+ json_response = response.json()
except requests.exceptions.Timeout:
raise APIRequestFailure('API call timed out')
except requests.exceptions.ConnectionError:
raise APIRequestFailure('Could not connect to security service')
except (requests.exceptions.RequestException, ValueError):
- logger.exception('Failed to get layer data response for %s', layer.id)
+ logger.exception('Failed to get layer data response for %s', layer_id)
raise APIRequestFailure()
if response.status_code == 404:
return None
- return response.json()
+ return json_response
- def _call(self, relative_url, body=None, *args, **kwargs):
+ def _call(self, method, relative_url, params=None, body=None):
""" Issues an HTTP call to the sec API at the given relative URL.
This function disconnects from the database while awaiting a response
from the API server.
@@ -184,18 +243,21 @@ class SecurityScannerAPI(object):
raise Exception('Cannot call unconfigured security system')
api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
- url = urljoin(api_url, relative_url % args)
+ url = urljoin(api_url, relative_url)
client = self.config['HTTPCLIENT']
timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
- logger.debug('Looking up sec information: %s', url)
with CloseForLongOperation(self.config):
- if body is not None:
+ if method == 'POST':
logger.debug('POSTing security URL %s', url)
- return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self._keys,
+ return client.post(url, json=body, params=params, timeout=timeout, cert=self._keys,
verify=self._certificate)
+ elif method == 'DELETE':
+ logger.debug('DELETEing security URL %s', url)
+ return client.delete(url, params=params, timeout=timeout, cert=self._keys,
+ verify=self._certificate)
else:
logger.debug('GETing security URL %s', url)
- return client.get(url, params=kwargs, timeout=timeout, cert=self._keys,
+ return client.get(url, params=params, timeout=timeout, cert=self._keys,
verify=self._certificate)
diff --git a/util/secscan/notifier.py b/util/secscan/notifier.py
new file mode 100644
index 000000000..a6df8ed9a
--- /dev/null
+++ b/util/secscan/notifier.py
@@ -0,0 +1,103 @@
+import logging
+import sys
+
+from collections import defaultdict
+
+from app import secscan_api
+from data.model.tag import filter_tags_have_repository_event, get_matching_tags
+from data.database import (Image, ImageStorage, ExternalNotificationEvent, Repository,
+ RepositoryTag)
+from endpoints.notificationhelper import spawn_notification
+from util.secscan import PRIORITY_LEVELS
+from util.secscan.api import APIRequestFailure
+
+logger = logging.getLogger(__name__)
+
+
+def process_notification_data(notification_data):
+ """ Processes the given notification data to spawn vulnerability notifications as necessary.
+ Returns whether the processing succeeded.
+ """
+ new_data = notification_data['New']
+ old_data = notification_data.get('Old', {})
+
+ new_vuln = new_data['Vulnerability']
+ old_vuln = old_data.get('Vulnerability', {})
+
+ new_layer_ids = set(new_data.get('LayersIntroducingVulnerability', []))
+ old_layer_ids = set(old_data.get('LayersIntroducingVulnerability', []))
+
+ new_severity = PRIORITY_LEVELS.get(new_vuln.get('Severity', 'Unknown'), {'index': sys.maxint})
+ old_severity = PRIORITY_LEVELS.get(old_vuln.get('Severity', 'Unknown'), {'index': sys.maxint})
+
+ # By default we only notify the new layers that are affected by the vulnerability. If, however,
+ # the severity of the vulnerability has increased, we need to notify *all* layers, as we might
+ # need to send new notifications for older layers.
+ notify_layers = new_layer_ids - old_layer_ids
+ if new_severity['index'] < old_severity['index']:
+ notify_layers = new_layer_ids | old_layer_ids
+
+ if not notify_layers:
+ # Nothing more to do.
+ return True
+
+ # Lookup the external event for when we have vulnerabilities.
+ event = ExternalNotificationEvent.get(name='vulnerability_found')
+
+ # For each layer, retrieving the matching tags and join with repository to determine which
+ # require new notifications.
+ tag_map = defaultdict(set)
+ repository_map = {}
+ cve_id = new_vuln['Name']
+
+ # Find all tags that contain the layer(s) introducing the vulnerability,
+ # in repositories that have the event setup.
+ for layer_id in notify_layers:
+ # Split the layer ID into its Docker Image ID and storage ID.
+ (docker_image_id, storage_uuid) = layer_id.split('.', 2)
+
+ # Find the matching tags.
+ matching = get_matching_tags(docker_image_id, storage_uuid, RepositoryTag, Repository,
+ Image, ImageStorage)
+ tags = list(filter_tags_have_repository_event(matching, event))
+
+ check_map = {}
+ for tag in tags:
+ # Verify that the tag's root image has the vulnerability.
+ tag_layer_id = '%s.%s' % (tag.image.docker_image_id, tag.image.storage.uuid)
+ logger.debug('Checking if layer %s is vulnerable to %s', tag_layer_id, cve_id)
+
+ if not tag_layer_id in check_map:
+ try:
+ is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id)
+ except APIRequestFailure:
+ return False
+
+ check_map[tag_layer_id] = is_vulerable
+
+ logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id,
+ check_map[tag_layer_id])
+
+ if check_map[tag_layer_id]:
+ # Add the vulnerable tag to the list.
+ tag_map[tag.repository_id].add(tag.name)
+ repository_map[tag.repository_id] = tag.repository
+
+ # For each of the tags found, issue a notification.
+ for repository_id in tag_map:
+ tags = tag_map[repository_id]
+ event_data = {
+ 'tags': list(tags),
+ 'vulnerability': {
+ 'id': cve_id,
+ 'description': new_vuln.get('Description', None),
+ 'link': new_vuln.get('Link', None),
+ 'priority': new_severity['title'],
+ 'has_fix': 'FixedIn' in new_vuln,
+ },
+ }
+
+ spawn_notification(repository_map[repository_id], 'vulnerability_found', event_data)
+
+ return True
+
diff --git a/workers/queueworker.py b/workers/queueworker.py
index 2895e31e7..7cc223a22 100644
--- a/workers/queueworker.py
+++ b/workers/queueworker.py
@@ -56,10 +56,11 @@ class QueueWorker(Worker):
logger.debug('Disconnecting from database.')
db.close()
- def extend_processing(self, seconds_from_now):
+ def extend_processing(self, seconds_from_now, updated_data=None):
with self._current_item_lock:
if self.current_queue_item is not None:
- self._queue.extend_processing(self.current_queue_item, seconds_from_now)
+ self._queue.extend_processing(self.current_queue_item, seconds_from_now,
+ updated_data=updated_data)
def run_watchdog(self):
logger.debug('Running watchdog.')
diff --git a/workers/security_notification_worker.py b/workers/security_notification_worker.py
index 603633343..a5e6d0480 100644
--- a/workers/security_notification_worker.py
+++ b/workers/security_notification_worker.py
@@ -1,80 +1,45 @@
-import json
import logging
import time
-
-from collections import defaultdict
+import json
import features
from app import secscan_notification_queue, secscan_api
-from data import model
-from data.model.tag import filter_tags_have_repository_event, get_matching_tags
-from data.database import (Image, ImageStorage, ExternalNotificationEvent,
- Repository, RepositoryNotification, RepositoryTag)
-from endpoints.notificationhelper import spawn_notification
-from workers.queueworker import QueueWorker
-
+from workers.queueworker import QueueWorker, JobException
+from util.secscan.notifier import process_notification_data
logger = logging.getLogger(__name__)
+_EXTENDED_SECONDS = 600
class SecurityNotificationWorker(QueueWorker):
def process_queue_item(self, data):
- cve_id = data['Name']
- vulnerability = data['Content']['Vulnerability']
- priority = vulnerability['Priority']
+ notification_name = data['Name']
+ current_page = data.get('page', None)
- # Lookup the external event for when we have vulnerabilities.
- event = ExternalNotificationEvent.get(name='vulnerability_found')
+ while True:
+ (response_data, should_retry) = secscan_api.get_notification(notification_name)
+ if response_data is None:
+ if should_retry:
+ raise JobException()
+ else:
+ # Return to mark the job as "complete", as we'll never be able to finish it.
+ logger.error('Failed to handle security notification %s', notification_name)
+ return
- # For each layer, retrieving the matching tags and join with repository to determine which
- # require new notifications.
- tag_map = defaultdict(set)
- repository_map = {}
+ notification_data = response_data['Notification']
+ if not process_notification_data(notification_data):
+ raise JobException()
- # Find all tags that contain the layer(s) introducing the vulnerability,
- # in repositories that have the event setup.
- content = data['Content']
- layer_ids = content.get('NewIntroducingLayersIDs', content.get('IntroducingLayersIDs', []))
- for layer_id in layer_ids:
- (docker_image_id, storage_uuid) = layer_id.split('.', 2)
+ # Check for a next page of results. If none, we're done.
+ if 'NextPage' not in notification_data:
+ return
- matching = get_matching_tags(docker_image_id, storage_uuid, RepositoryTag, Repository,
- Image, ImageStorage)
- tags = list(filter_tags_have_repository_event(matching, event))
-
- check_map = {}
- for tag in tags:
- # Verify that the tag's root image has the vulnerability.
- tag_layer_id = '%s.%s' % (tag.image.docker_image_id, tag.image.storage.uuid)
- logger.debug('Checking if layer %s is vulnerable to %s', tag_layer_id, cve_id)
-
- if not tag_layer_id in check_map:
- is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id)
- check_map[tag_layer_id] = is_vulerable
-
- logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id,
- check_map[tag_layer_id])
-
- if check_map[tag_layer_id]:
- # Add the vulnerable tag to the list.
- tag_map[tag.repository_id].add(tag.name)
- repository_map[tag.repository_id] = tag.repository
-
- # For each of the tags found, issue a notification.
- for repository_id in tag_map:
- tags = tag_map[repository_id]
- event_data = {
- 'tags': list(tags),
- 'vulnerability': {
- 'id': data['Name'],
- 'description': vulnerability['Description'],
- 'link': vulnerability['Link'],
- 'priority': priority,
- },
- }
-
- spawn_notification(repository_map[repository_id], 'vulnerability_found', event_data)
+ # Otherwise, save the next page token into the queue item (so we can pick up from here if
+ # something goes wrong in the next loop iteration), and continue.
+ current_page = notification_data['NextPage']
+ data['page'] = current_page
+ self.extend_processing(_EXTENDED_SECONDS, json.dumps(data))
if __name__ == '__main__':
From c7904db30d2cdb5a85788421d7d50a40bdea4999 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Mon, 22 Feb 2016 12:21:22 -0500
Subject: [PATCH 07/12] v2: always send www-authn headers on unauthorized
Fixes #1254.
---
endpoints/v2/__init__.py | 3 +++
test/registry_tests.py | 5 ++++-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/endpoints/v2/__init__.py b/endpoints/v2/__init__.py
index 1624b3376..b02b1e541 100644
--- a/endpoints/v2/__init__.py
+++ b/endpoints/v2/__init__.py
@@ -30,7 +30,10 @@ def handle_registry_v2_exception(error):
response = jsonify({
'errors': [error.as_dict()]
})
+
response.status_code = error.http_status_code
+ if response.status_code == 401:
+ response.headers.extend(get_auth_headers())
logger.debug('sending response: %s', response.get_data())
return response
diff --git a/test/registry_tests.py b/test/registry_tests.py
index c6315e053..2030012ce 100644
--- a/test/registry_tests.py
+++ b/test/registry_tests.py
@@ -1319,7 +1319,10 @@ class V2RegistryTests(V2RegistryPullMixin, V2RegistryPushMixin, RegistryTestsMix
self.assertEquals(len(data['tags']), 1)
# Try to get tags before a repo exists.
- self.conduct('GET', '/v2/devtable/doesnotexist/tags/list', auth='jwt', expected_code=401)
+ response = self.conduct('GET', '/v2/devtable/doesnotexist/tags/list', auth='jwt', expected_code=401)
+
+ # Assert 401s to non-auth endpoints also get the WWW-Authenticate header.
+ self.assertIn('WWW-Authenticate', response.headers)
def test_one_five_blacklist(self):
self.conduct('GET', '/v2/', expected_code=404, user_agent='Go 1.1 package http')
From ae9140caaecf9c75f33c71fde1f6ad1355053066 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <josephschorr@users.noreply.github.com>
Date: Mon, 22 Feb 2016 18:39:04 -0500
Subject: [PATCH 08/12] Implement new vulnerabilities and packages tabs.
Fixes https://github.com/coreos-inc/design/issues/268
---
static/css/core-ui.css | 28 ++
.../css/directives/ui/image-feature-view.css | 109 ++++++
.../ui/image-vulnerability-view.css | 169 ++++++++++
.../css/directives/ui/nvd-vectors-display.css | 50 +++
static/directives/image-feature-view.html | 148 +++++++++
.../directives/image-vulnerability-view.html | 153 +++++++++
static/directives/nvd-vectors-display.html | 8 +
.../directives/repo-view/repo-panel-tags.html | 6 +-
static/js/directives/ui/image-feature-view.js | 311 ++++++++++++++++++
.../directives/ui/image-vulnerability-view.js | 252 ++++++++++++++
.../js/directives/ui/nvd-vectors-display.js | 35 ++
static/js/pages/image-view.js | 63 +---
static/js/services/image-metadata-service.js | 29 ++
static/js/services/vulnerability-service.js | 226 ++++++++++++-
static/partials/image-view.html | 117 +------
util/secscan/__init__.py | 14 +
16 files changed, 1547 insertions(+), 171 deletions(-)
create mode 100644 static/css/directives/ui/image-feature-view.css
create mode 100644 static/css/directives/ui/image-vulnerability-view.css
create mode 100644 static/css/directives/ui/nvd-vectors-display.css
create mode 100644 static/directives/image-feature-view.html
create mode 100644 static/directives/image-vulnerability-view.html
create mode 100644 static/directives/nvd-vectors-display.html
create mode 100644 static/js/directives/ui/image-feature-view.js
create mode 100644 static/js/directives/ui/image-vulnerability-view.js
create mode 100644 static/js/directives/ui/nvd-vectors-display.js
diff --git a/static/css/core-ui.css b/static/css/core-ui.css
index f1a7499d6..a9a260b47 100644
--- a/static/css/core-ui.css
+++ b/static/css/core-ui.css
@@ -1048,12 +1048,18 @@ a:focus {
padding-left: 6px;
padding-right: 0px;
color: #aaa;
+ text-align: center;
+ max-width: 20px;
}
.co-table td.caret-col i.fa {
cursor: pointer;
}
+.co-table td.caret-col i.fa.fa-caret-down {
+ color: black;
+}
+
.co-table .add-row-spacer td {
padding: 5px;
}
@@ -1177,6 +1183,28 @@ a:focus {
background: #F6FCFF;
}
+.co-filter-box {
+ position: relative;;
+}
+
+.co-filter-box .filter-message {
+ display: inline-block;
+ position: absolute;
+ left: -220px;
+ top: 7px;
+ color: #ccc;
+}
+
+.co-filter-box .filter-options {
+ margin-top: 4px;
+ font-size: 14px;
+ text-align: right;
+}
+
+.co-filter-box .filter-options input {
+ margin-right: 6px;
+}
+
.co-check-bar {
margin-bottom: 10px;
}
diff --git a/static/css/directives/ui/image-feature-view.css b/static/css/directives/ui/image-feature-view.css
new file mode 100644
index 000000000..556a97054
--- /dev/null
+++ b/static/css/directives/ui/image-feature-view.css
@@ -0,0 +1,109 @@
+.image-feature-view-element .donut-icon {
+ position: absolute;
+ top: 60px;
+ left: 95px;
+ font-size: 80px;
+ text-align: center;
+ color: #EAEAEA;
+ margin-left: -6px;
+}
+
+.image-feature-view-element .no-vulns {
+ color: #2FC98E;
+}
+
+.image-feature-view-element .no-vulns i.fa {
+ margin-right: 6px;
+}
+
+.image-feature-view-element .security-header {
+ margin-top: -4px;
+ margin-bottom: 30px;
+ padding-bottom: 30px;
+ border-bottom: 1px solid #eee;
+}
+
+.image-feature-view-element .donut-col {
+ padding-top: 20px;
+ text-align: center;
+}
+
+.image-feature-view-element #featureDonutChart {
+ display: inline-block;
+}
+
+.image-feature-view-element .summary-col {
+ font-size: 20px;
+ padding-top: 40px;
+}
+
+.image-feature-view-element .summary-col .title-item {
+ font-size: 24px;
+ margin-bottom: 40px;
+}
+
+.image-feature-view-element .summary-list {
+ text-align: left;
+ list-style: none;
+}
+
+.image-feature-view-element .summary-list i.fa {
+ margin-right: 10px;
+}
+
+.image-feature-view-element .summary-list strong {
+ text-align: right;
+ width: 40px;
+ display: inline-block;
+ margin-right: 6px;
+}
+
+.image-feature-view-element .co-table .empty {
+ color: #ddd;
+}
+
+.image-feature-view-element .co-table .single-col {
+ width: 12.5%;
+}
+
+.image-feature-view-element .co-table .double-col {
+ width: 25%;
+}
+
+.image-feature-view-element .co-table .impact-col {
+ text-align: center;
+ width: 130px;
+}
+
+.image-feature-view-element .co-table .image-col {
+ white-space: nowrap;
+}
+
+.image-feature-view-element .co-table .image-col .fa {
+ margin-left: 6px;
+ opacity: 0.5;
+}
+
+@media (max-width: 767px) {
+ .image-feature-view-element .co-table .single-col {
+ width: auto !important;
+ }
+}
+
+.image-feature-view-element .dockerfile-command {
+ cursor: default;
+}
+
+.image-feature-view-element .dockerfile-command .command-title {
+ font-size: 12px;
+ max-width: 297px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ display: inline-block;
+ white-space: nowrap;
+ vertical-align: middle;
+}
+
+.image-feature-view-element .vuln-summary i.fa {
+ margin-right: 6px;
+}
diff --git a/static/css/directives/ui/image-vulnerability-view.css b/static/css/directives/ui/image-vulnerability-view.css
new file mode 100644
index 000000000..b14ec0d72
--- /dev/null
+++ b/static/css/directives/ui/image-vulnerability-view.css
@@ -0,0 +1,169 @@
+.image-vulnerability-view-element .donut-icon {
+ position: absolute;
+ top: 70px;
+ left: 95px;
+ font-size: 80px;
+ text-align: center;
+ color: #EAEAEA;
+ margin-left: -6px;
+}
+
+.image-vulnerability-view-element .security-header {
+ margin-top: -4px;
+ margin-bottom: 30px;
+ padding-bottom: 30px;
+ border-bottom: 1px solid #eee;
+}
+
+.image-vulnerability-view-element .donut-col {
+ padding-top: 20px;
+ text-align: center;
+}
+
+.image-vulnerability-view-element #vulnDonutChart {
+ display: inline-block;
+}
+
+.image-vulnerability-view-element .summary-col {
+ font-size: 20px;
+ padding-top: 40px;
+}
+
+.image-vulnerability-view-element .summary-col .title-item {
+ font-size: 24px;
+ margin-bottom: 40px;
+}
+
+.image-vulnerability-view-element .summary-list {
+ text-align: left;
+ list-style: none;
+}
+
+.image-vulnerability-view-element .summary-list i.fa {
+ margin-right: 10px;
+}
+
+.image-vulnerability-view-element .summary-list strong {
+ text-align: right;
+ width: 40px;
+ display: inline-block;
+ margin-right: 6px;
+}
+
+.image-vulnerability-view-element .dockerfile-command {
+ cursor: default;
+}
+
+.image-vulnerability-view-element .dockerfile-command .command-title {
+ font-size: 12px;
+ max-width: 297px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ display: inline-block;
+ white-space: nowrap;
+ vertical-align: middle;
+}
+
+.image-vulnerability-view-element .co-table .empty {
+ color: #ddd;
+}
+
+.image-vulnerability-view-element .co-table .single-col {
+ width: 15%;
+}
+
+.image-vulnerability-view-element .co-table .double-col {
+ width: 30%;
+}
+
+.image-vulnerability-view-element .co-table .impact-col {
+ text-align: center;
+ width: 130px;
+}
+
+.image-vulnerability-view-element .co-table .nowrap-col {
+ white-space: nowrap;
+}
+
+.image-vulnerability-view-element .co-table .image-col {
+ white-space: nowrap;
+}
+
+.image-vulnerability-view-element .co-table .image-col .fa {
+ margin-left: 6px;
+ opacity: 0.5;
+}
+
+@media (max-width: 767px) {
+ .image-vulnerability-view-element .co-table .single-col {
+ width: auto !important;
+ }
+}
+
+.image-vulnerability-view-element .fixed-in-version:before {
+ font-family: FontAwesome;
+ content: '\f0a9';
+ margin-right: 6px;
+}
+
+.image-vulnerability-view-element .fixed-in-version {
+ color: rgb(47, 201, 142);
+}
+
+.image-vulnerability-view-element .cvss-text {
+ display: inline-block;
+ width: 20px;
+ text-align: right;
+}
+
+.image-vulnerability-view-element .vulnerability-priority-view {
+ margin-left: 10px;
+}
+
+.image-vulnerability-view-element .cvss {
+ display: inline-block;
+ width: 100px;
+ height: 10px;
+ vertical-align: middle;
+ margin-left: 8px;
+ background: #eee;
+ position: relative;;
+}
+
+.image-vulnerability-view-element .cvss span {
+ display: inline-block;
+ position: absolute;
+ top: 0px;
+ left: 0px;
+ bottom: 0px;
+}
+
+.image-vulnerability-view-element .expansion-col {
+ padding-top: 20px;
+ padding-bottom: 20px;
+}
+
+.image-vulnerability-view-element .subtitle {
+ color: #999;
+ font-size: 90%;
+ text-transform: uppercase;
+ font-weight: 300;
+ padding-top: 0!important;
+ text-align: left;
+ margin-bottom: 6px;
+}
+
+
+.image-vulnerability-view-element .expand-link {
+ color: black !important;
+}
+
+.image-vulnerability-view-element .external-link {
+ margin-left: 10px;
+ font-size: 12px;
+}
+
+.image-vulnerability-view-element .description {
+ display: inline-block;
+ max-width: 1000px;
+}
diff --git a/static/css/directives/ui/nvd-vectors-display.css b/static/css/directives/ui/nvd-vectors-display.css
new file mode 100644
index 000000000..cfec11cdb
--- /dev/null
+++ b/static/css/directives/ui/nvd-vectors-display.css
@@ -0,0 +1,50 @@
+.nvd-vectors-display-element dl {
+ display: inline-block;
+ vertical-align: top;
+ max-width: 200px;
+ padding: 10px;
+}
+
+.nvd-vectors-display-element dt {
+ margin-bottom: 10px;
+ cursor: pointer;
+}
+
+.nvd-vectors-display-element dd {
+ cursor: pointer;
+}
+
+.nvd-vectors-display-element dd:before {
+ display: inline-block;
+ margin-right: 4px;
+ font-family: FontAwesome;
+ content: '\f111';
+}
+
+.nvd-vectors-display-element dd.not-current-vector {
+ color: #ddd;
+}
+
+.nvd-vectors-display-element dd.current-vector.high:before {
+ content: '\f071';
+}
+
+.nvd-vectors-display-element dd.current-vector.medium:before {
+ content: '\f06a';
+}
+
+.nvd-vectors-display-element dd.current-vector.low:before {
+ content: '\f056';
+}
+
+.nvd-vectors-display-element .current-vector.high {
+ color: rgb(214, 68, 86);
+}
+
+.nvd-vectors-display-element .current-vector.medium {
+ color: rgb(252, 166, 87);
+}
+
+.nvd-vectors-display-element .current-vector.low {
+ color: rgb(47, 201, 142);
+}
\ No newline at end of file
diff --git a/static/directives/image-feature-view.html b/static/directives/image-feature-view.html
new file mode 100644
index 000000000..b23c06589
--- /dev/null
+++ b/static/directives/image-feature-view.html
@@ -0,0 +1,148 @@
+<div class="image-feature-view-element">
+ <!-- Not scanned -->
+ <div class="empty" ng-if="securityStatus == 'queued'">
+ <div class="empty-primary-msg">This image has not been indexed yet</div>
+ <div class="empty-secondary-msg">
+ Please try again in a few minutes.
+ </div>
+ </div>
+
+ <!-- Unable to scan -->
+ <div class="empty" ng-if="securityStatus == 'failed'">
+ <div class="empty-primary-msg">This image could not be indexed</div>
+ <div class="empty-secondary-msg">
+ Our security scanner was unable to index this image.
+ </div>
+ </div>
+
+ <!-- Scanned -->
+ <div ng-if="securityStatus == 'scanned'">
+ <!-- Header -->
+ <div class="security-header row">
+ <div class="donut-col col-md-3">
+ <div id="featureDonutChart" style="position: relative;">
+ <svg style="height:250px; width:250px"></svg>
+ <span class="donut-icon">
+ <i class="fa ci-package"></i>
+ </span>
+ </div>
+ </div>
+ <div class="summary-col col-md-9">
+ <ul class="summary-list">
+ <li class="title-item">Quay Security Scanner has resolved <strong>{{ securityFeatures.length }}</strong> packages.</li>
+ <li ng-repeat="priority in featureBreakdown">
+ <span ng-if="priority.label != 'None'">
+ <i class="fa ci-package" ng-style="{'color': priority.color}"></i> <strong>{{ priority.value }}</strong> packages with {{ priority.label }}-level vulnerabilities.
+ </span>
+ <span ng-if="priority.label == 'None'" style="margin-top: 20px; display: inline-block;">
+ <i class="fa ci-package" ng-style="{'color': priority.color}"></i> <strong>{{ priority.value }}</strong> packages with no vulnerabilities.
+ </span>
+ </li>
+ </ul>
+ </div>
+ </div>
+
+ <!-- Filter -->
+ <span class="co-filter-box">
+ <span class="filter-message" ng-if="options.featureFilter">
+ Showing {{ orderedFeatures.entries.length }} of {{ securityFeatures.length }} packages
+ </span>
+ <input class="form-control" type="text" ng-model="options.featureFilter" placeholder="Filter Packages...">
+ </span>
+ <h3>Image Packages</h3>
+
+ <!-- Table -->
+ <table class="co-table">
+ <thead>
+ <td ng-class="tablePredicateClass('name', options.predicate, options.reverse)">
+ <a href="javascript:void(0)" ng-click="orderBy('name')">Package Name</a>
+ </td>
+ <td class="hidden-xs">
+ Package Version
+ </td>
+ <td ng-class="tablePredicateClass('score', options.predicate, options.reverse)">
+ <a href="javascript:void(0)" ng-click="orderBy('score')">Vulnerabilities</a>
+ </td>
+ <td class="hidden-xs hidden-sm hidden-md"
+ ng-class="tablePredicateClass('leftoverScore', options.predicate, options.reverse)"
+ data-title="Identified vulnerabilities remaining after the package is upgraded to the latest version"
+ data-container="body" bs-tooltip>
+ <a href="javascript:void(0)" ng-click="orderBy('leftoverScore')">Remaining after upgrade</a>
+ </td>
+ <td ng-class="tablePredicateClass('fixableScore', options.predicate, options.reverse)">
+ <a href="javascript:void(0)" ng-click="orderBy('fixableScore')"
+ data-title="Delta of the severity of vulnerabilities in the package before->after upgrading" data-container="body" bs-tooltip>Upgrade impact</a>
+ </td>
+ <td class="hidden-xs hidden-sm hidden-md">
+ Introduced In Image
+ </td>
+ <td class="hidden-xs options-col"></td>
+ </thead>
+ <tbody ng-repeat="feature in orderedFeatures.visibleEntries" bindonce>
+ <tr>
+ <td class="single-col">
+ <span bo-text="feature.name"></span>
+ </td>
+ <td class="single-col hidden-xs">
+ <span bo-text="feature.version"></span>
+ </td>
+ <td>
+ <span class="no-vulns" bo-if="feature.vulnCount == 0">
+ <i class="fa fa-check-circle"></i>None Detected
+ </span>
+ <span class="vuln-summary" bo-if="feature.vulnCount != 0">
+ <span ng-style="{'color': feature.primarySeverity.color}">
+ <i class="fa fa fa-exclamation-triangle"></i>
+ {{ feature.primarySeverity.count }}
+ {{ feature.primarySeverity.title }}
+ </span>
+ <span bo-if="feature.vulnCount - feature.primarySeverity.count > 0">
+ + {{ feature.vulnCount - feature.primarySeverity.count }} additional
+ </span>
+ </span>
+ </td>
+ <td class="hidden-xs hidden-sm hidden-md">
+ <span class="empty" bo-if="feature.vulnCount == 0">
+ (N/A)
+ </span>
+ <span class="no-vulns" bo-if="feature.vulnCount != 0 && feature.leftoverBreakdown.length == 0">
+ <i class="fa fa-arrow-circle-right"></i>
+ All identified vulnerabilities fixed
+ </span>
+ <span class="vuln-summary" bo-if="feature.vulnCount != 0 && feature.leftoverBreakdown.length != 0">
+ <span ng-style="{'color': feature.primaryLeftover.color}">
+ <i class="fa fa-arrow-circle-right"></i>
+ {{ feature.primaryLeftover.count }}
+ {{ feature.primaryLeftover.title }}
+ </span>
+ <span bo-if="feature.leftoverCount - feature.primaryLeftover.count > 0">
+ + {{ feature.leftoverCount - feature.primaryLeftover.count }} additional
+ </span>
+ </span>
+ </td>
+ <td class="impact-col">
+ <span class="empty" bo-if="feature.vulnCount == 0">
+ (N/A)
+ </span>
+ <span bo-if="feature.vulnCount > 0">
+ <span class="strength-indicator" value="feature.fixableScore" maximum="highestFixableScore"
+ log-base="2"></span>
+ </span>
+ </td>
+ <td class="double-col image-col hidden-xs hidden-sm hidden-md">
+ <span data-title="{{ feature.imageCommand }}" bs-tooltip>
+ <span class="dockerfile-command" command="feature.imageCommand"></span>
+ </span>
+ <a href="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ feature.imageId }}"><i class="fa fa-archive"></i></a>
+ </td>
+ <td></td>
+ </tr>
+ </tbody>
+ </table>
+ <div class="empty" ng-if="securityFeatures.length && !orderedFeatures.entries.length"
+ style="margin-top: 20px;">
+ <div class="empty-primary-msg">No matching packages found.</div>
+ <div class="empty-secondary-msg">Try expanding your filtering terms.</div>
+ </div>
+ </div>
+</div>
\ No newline at end of file
diff --git a/static/directives/image-vulnerability-view.html b/static/directives/image-vulnerability-view.html
new file mode 100644
index 000000000..919dd1966
--- /dev/null
+++ b/static/directives/image-vulnerability-view.html
@@ -0,0 +1,153 @@
+<div class="image-vulnerability-view-element">
+ <!-- Not scanned -->
+ <div class="empty" ng-if="securityStatus == 'queued'">
+ <div class="empty-primary-msg">This image has not been indexed yet</div>
+ <div class="empty-secondary-msg">
+ Please try again in a few minutes.
+ </div>
+ </div>
+
+ <!-- Unable to scan -->
+ <div class="empty" ng-if="securityStatus == 'failed'">
+ <div class="empty-primary-msg">This image could not be indexed</div>
+ <div class="empty-secondary-msg">
+ Our security scanner was unable to index this image.
+ </div>
+ </div>
+
+ <!-- Scanned -->
+ <div ng-if="securityStatus == 'scanned'">
+ <!-- Header -->
+ <div class="security-header row">
+ <div class="donut-col col-md-3">
+ <div id="vulnDonutChart" style="position: relative;">
+ <svg style="height:250px; width:250px"></svg>
+ <span class="donut-icon">
+ <i class="fa fa-bug"></i>
+ </span>
+ </div>
+ </div>
+ <div class="summary-col col-md-9">
+ <ul class="summary-list" ng-if="priorityBreakdown.length">
+ <li class="title-item">Quay Security Scanner has detected <strong>{{ securityVulnerabilities.length }}</strong> vulnerabilities.</li>
+
+ <li ng-repeat="priority in priorityBreakdown">
+ <i class="fa fa-exclamation-triangle" ng-style="{'color': priority.color}"></i> <strong>{{ priority.value }}</strong> {{ priority.label }}-level vulnerabilities.
+ </li>
+ </ul>
+
+ <div ng-if="!priorityBreakdown.length">
+ <li class="title-item">Quay Security Scanner has detected no vulnerabilities in this image.</li>
+ </div>
+ </div>
+ </div>
+
+ <!-- Filter -->
+ <span class="co-filter-box">
+ <span class="filter-message" ng-if="options.vulnFilter || options.fixableVulns">
+ Showing {{ orderedVulnerabilities.entries.length }} of {{ securityVulnerabilities.length }} Vulnerabilities
+ </span>
+ <input class="form-control" type="text" ng-model="options.vulnFilter" placeholder="Filter Vulnerabilities...">
+ <div class="filter-options">
+ <label><input type="checkbox" ng-model="options.fixableVulns">Only display vulnerabilities with fixes</label>
+ </div>
+ </span>
+ <h3>Image Vulnerabilities</h3>
+
+ <!-- Table -->
+ <table class="co-table">
+ <thead>
+ <td class="caret-col"></td>
+ <td ng-class="tablePredicateClass('name', options.predicate, options.reverse)">
+ <a href="javascript:void(0)" ng-click="orderBy('name')">CVE</a>
+ </td>
+ <td ng-class="tablePredicateClass('score', options.predicate, options.reverse)">
+ <a href="javascript:void(0)" ng-click="orderBy('score')">CVSS / Severity</a>
+ </td>
+ <td class="hidden-xs" ng-class="tablePredicateClass('featureName', options.predicate, options.reverse)">
+ <a href="javascript:void(0)" ng-click="orderBy('featureName')">Package</a>
+ </td>
+ <td class="hidden-xs">Current version</td>
+ <td class="hidden-xs hidden-sm">Fixed in version</td>
+ </td>
+ <td class="hidden-xs hidden-sm hidden-md">Introduced in image</td>
+ <td class="hidden-xs options-col"></td>
+ </thead>
+ <tbody ng-repeat="vuln in orderedVulnerabilities.visibleEntries" bindonce>
+ <tr>
+ <td class="caret-col">
+ <span ng-click="toggleDetails(vuln)">
+ <i class="fa"
+ ng-class="vuln.expanded ? 'fa-caret-down' : 'fa-caret-right'"
+ data-title="View Details" bs-tooltip></i>
+ </span>
+ </td>
+ <td class="single-col nowrap-col">
+ <a href="javascript:void(0)" bo-text="vuln.name" ng-click="toggleDetails(vuln)" class="expand-link"></a>
+ <a href="{{ vuln.link }}" class="external-link hidden-xs hidden-sm hidden-md" target="_blank">
+ <i class="fa fa-link"></i>
+ </a>
+ </td>
+ <td class="single-col nowrap-col">
+ <span bo-if="vuln.metadata.NVD.CVSSv2.Score">
+ <span class="cvss-text" bo-text="vuln.metadata.NVD.CVSSv2.Score"></span>
+ <span class="cvss"><span bo-style="{'width': (vuln.metadata.NVD.CVSSv2.Score * 10) + '%', 'background-color': getCVSSColor(vuln.metadata.NVD.CVSSv2.Score)}"></span>
+ </span>
+ </span>
+ <span bo-if="!vuln.metadata.NVD.CVSSv2.Score">
+ <span class="vulnerability-priority-view" priority="vuln.severity"></span>
+ </span>
+ </td>
+ <td class="single-col hidden-xs"><span bo-text="vuln.featureName"></span></td>
+ <td class="single-col hidden-xs hidden-sm"><span bo-text="vuln.introducedInVersion"></span></td>
+ <td class="single-col hidden-xs">
+ <span class="empty" bo-if="!vuln.fixedInVersion">(None)</span>
+ <span class="fixed-in-version" bo-if="vuln.fixedInVersion" bo-text="vuln.fixedInVersion"></span>
+ </td>
+ <td class="double-col image-col hidden-xs hidden-sm hidden-md">
+ <span data-title="{{ vuln.imageCommand }}" bs-tooltip>
+ <span class="dockerfile-command" command="vuln.imageCommand"></span>
+ </span>
+ <a href="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ vuln.imageId }}"><i class="fa fa-archive"></i></a>
+ </td>
+ <td></td>
+ </tr>
+ <tr ng-if="vuln.expanded">
+ <td class="expansion-col" colspan="8">
+ <div class="visible-xs" style="margin-bottom: 20px">
+ <div class="subtitle">Summary</div>
+ <table>
+ <tr><td>Package:</td><td><span bo-text="vuln.featureName"></span></td></tr>
+ <tr><td>Introduced in version:</td><td><span bo-text="vuln.introducedInVersion"></span></td></tr>
+ <tr>
+ <td>Fixed in version:</td>
+ <td>
+ <span class="empty" bo-if="!vuln.fixedInVersion">(None)</span>
+ <span class="fixed-in-version" bo-if="vuln.fixedInVersion" bo-text="vuln.fixedInVersion"></span>
+ </td>
+ </tr>
+ <tr>
+ <td>Introduced in Image:</td>
+ <td><span class="image-link" repository="repository" image-id="vuln.imageId"></span></td>
+ </tr>
+ </table>
+ </div>
+
+ <div class="vectors" bo-if="vuln.metadata.NVD.CVSSv2.Vectors">
+ <div class="subtitle">Vectors</div>
+ <div class="nvd-vectors-display" vectors="{{ vuln.metadata.NVD.CVSSv2.Vectors }}"></div>
+ </div>
+
+ <div class="subtitle">Description</div>
+ <span class="description" bo-text="vuln.description"></span>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ <div class="empty" ng-if="securityVulnerabilities.length && !orderedVulnerabilities.entries.length"
+ style="margin-top: 20px;">
+ <div class="empty-primary-msg">No matching vulnerabilities found.</div>
+ <div class="empty-secondary-msg">Try expanding your filtering terms.</div>
+ </div>
+ </div>
+</div>
\ No newline at end of file
diff --git a/static/directives/nvd-vectors-display.html b/static/directives/nvd-vectors-display.html
new file mode 100644
index 000000000..973f14a0f
--- /dev/null
+++ b/static/directives/nvd-vectors-display.html
@@ -0,0 +1,8 @@
+<div class="nvd-vectors-display-element">
+ <dl ng-repeat="vector in parsedVectors" bindonce>
+ <dt bo-text="getVectorTitle(vector)" data-title="{{ getVectorDescription(vector) }}" data-container="body" bs-tooltip></dt>
+ <dd bo-class="getVectorClasses(option, vector)" ng-repeat="option in getVectorOptions(vector)" data-container="body" data-title="{{ option.description }}" bs-tooltip>
+ {{ option.title }}
+ </dd>
+ </dl>
+</div>
\ No newline at end of file
diff --git a/static/directives/repo-view/repo-panel-tags.html b/static/directives/repo-view/repo-panel-tags.html
index 1b7c04726..b582f0521 100644
--- a/static/directives/repo-view/repo-panel-tags.html
+++ b/static/directives/repo-view/repo-panel-tags.html
@@ -152,7 +152,7 @@
data-title="The image for this tag has no vulnerabilities as found in our database"
bs-tooltip
bindonce>
- <a bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=security">
+ <a bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=vulnerabilities">
<i class="fa fa-check-circle"></i>
Passed
</a>
@@ -162,7 +162,7 @@
<span ng-if="getTagVulnerabilities(tag).status == 'scanned' && getTagVulnerabilities(tag).hasVulnerabilities"
ng-class="getTagVulnerabilities(tag).highestVulnerability.Priority"
class="has-vulns" bindonce>
- <a class="vuln-link" bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=security"
+ <a class="vuln-link" bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=vulnerabilities"
data-title="The image for this tag has {{ getTagVulnerabilities(tag).highestVulnerability.Count }} {{ getTagVulnerabilities(tag).highestVulnerability.Priority }} level vulnerabilities"
bs-tooltip>
<span class="highest-vuln">
@@ -176,7 +176,7 @@
+ {{ getTagVulnerabilities(tag).vulnerabilities.length - getTagVulnerabilities(tag).highestVulnerability.Count }} others
</span>
</a>
- <a bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=security" style="display: inline-block; margin-left: 6px;">
+ <a bo-href-i="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ tag.image_id }}?tab=vulnerabilities" style="display: inline-block; margin-left: 6px;">
More Info
</a>
</span>
diff --git a/static/js/directives/ui/image-feature-view.js b/static/js/directives/ui/image-feature-view.js
new file mode 100644
index 000000000..f0b02a6cd
--- /dev/null
+++ b/static/js/directives/ui/image-feature-view.js
@@ -0,0 +1,311 @@
+/**
+ * An element which displays the features of an image.
+ */
+angular.module('quay').directive('imageFeatureView', function () {
+ var directiveDefinitionObject = {
+ priority: 0,
+ templateUrl: '/static/directives/image-feature-view.html',
+ replace: false,
+ transclude: true,
+ restrict: 'C',
+ scope: {
+ 'repository': '=repository',
+ 'image': '=image',
+ 'isEnabled': '=isEnabled'
+ },
+ controller: function($scope, $element, Config, ApiService, VulnerabilityService, AngularViewArray, ImageMetadataService) {
+ var imageMap = null;
+
+ $scope.securityFeatures = [];
+ $scope.featureBreakdown = [];
+
+ $scope.options = {
+ 'featureFilter': null,
+ 'predicate': 'fixableScore',
+ 'reverse': false,
+ };
+
+ $scope.tablePredicateClass = function(name, predicate, reverse) {
+ if (name != predicate) {
+ return '';
+ }
+
+ return 'current ' + (reverse ? 'reversed' : '');
+ };
+
+ $scope.orderBy = function(predicate) {
+ if (predicate == $scope.options.predicate) {
+ $scope.options.reverse = !$scope.options.reverse;
+ return;
+ }
+
+ $scope.options.reverse = false;
+ $scope.options.predicate = predicate;
+ };
+
+ var buildOrderedFeatures = function() {
+ var features = $scope.securityFeatures.slice(0);
+ $scope.orderedFeatures = AngularViewArray.create();
+ features.forEach(function(v) {
+ var featureFilter = $scope.options.featureFilter;
+ if (featureFilter) {
+ if ((v['name'].indexOf(featureFilter) < 0) &&
+ (v['version'].indexOf(featureFilter) < 0) &&
+ (v['imageId'].indexOf(featureFilter) < 0)) {
+ return;
+ }
+ }
+
+ $scope.orderedFeatures.push(v);
+ });
+
+ $scope.orderedFeatures.entries.sort(function(a, b) {
+ var left = a[$scope.options['predicate']];
+ var right = b[$scope.options['predicate']];
+
+ if ($scope.options['predicate'] == 'score' ||
+ $scope.options['predicate'] == 'fixableScore' ||
+ $scope.options['predicate'] == 'leftoverScore') {
+ left = left * 1;
+ right = right * 1;
+ }
+
+ if (left == null) {
+ left = '0.00';
+ }
+
+ if (right == null) {
+ right = '0.00';
+ }
+
+ if (left == right) {
+ return 0;
+ }
+
+ return left > right ? -1 : 1;
+ });
+
+ if ($scope.options['reverse']) {
+ $scope.orderedFeatures.entries.reverse();
+ }
+
+ $scope.orderedFeatures.setVisible(true);
+ };
+
+ var buildChart = function() {
+ var chartData = $scope.featureBreakdown;
+ var colors = [];
+ for (var i = 0; i < chartData.length; ++i) {
+ colors.push(chartData[i].color);
+ }
+
+ nv.addGraph(function() {
+ var chart = nv.models.pieChart()
+ .x(function(d) { return d.label })
+ .y(function(d) { return d.value })
+ .margin({left: -10, right: -10, top: -10, bottom: -10})
+ .showLegend(false)
+ .showLabels(true)
+ .labelThreshold(.05)
+ .labelType("percent")
+ .donut(true)
+ .color(colors)
+ .donutRatio(0.5);
+
+ d3.select("#featureDonutChart svg")
+ .datum(chartData)
+ .transition()
+ .duration(350)
+ .call(chart);
+
+ return chart;
+ });
+ };
+
+ var buildFeatures = function(data) {
+ $scope.securityFeatures = [];
+ $scope.featureBreakdown = [];
+ $scope.highestFixableScore = -10000;
+
+ var severityMap = {};
+ var levels = VulnerabilityService.getLevels();
+
+ if (data && data.Layer && data.Layer.Features) {
+ data.Layer.Features.forEach(function(feature) {
+ var imageId = feature.AddedBy.split('.')[0];
+ feature_obj = {
+ 'name': feature.Name,
+ 'namespace': feature.Namespace,
+ 'version': feature.Version,
+ 'addedBy': feature.AddedBy,
+ 'imageId': imageId,
+ 'imageCommand': ImageMetadataService.getImageCommand($scope.image, imageId),
+ 'vulnCount': 0,
+ 'severityBreakdown': [],
+ 'fixableBreakdown': [],
+ 'score': 0,
+ 'fixableCount': 0,
+ 'leftoverCount': 0,
+ 'fixableScore': 0,
+ 'leftoverScore': 0,
+ 'unfixableCount': 0
+ }
+
+ if (feature.Vulnerabilities) {
+ var highestSeverity = null;
+ var localSeverityMap = {};
+ var localLeftoverMap = {};
+
+ feature.Vulnerabilities.forEach(function(vuln) {
+ var severity = VulnerabilityService.LEVELS[vuln['Severity']];
+ var score = severity.score;
+ if (vuln.Metadata && vuln.Metadata.NVD && vuln.Metadata.NVD.CVSSv2 && vuln.Metadata.NVD.CVSSv2.Score) {
+ score = vuln.Metadata.NVD.CVSSv2.Score;
+ severity = VulnerabilityService.getSeverityForCVSS(score);
+ }
+
+ var logScore = (Math.pow(2, score) + 0.1);
+ feature_obj['score'] += logScore;
+
+ if (vuln.FixedBy) {
+ feature_obj['fixableScore'] += logScore;
+ feature_obj['fixableCount']++;
+ } else {
+ feature_obj['leftoverCount']++;
+ feature_obj['leftoverScore'] += logScore;
+ }
+
+ if (highestSeverity == null) {
+ highestSeverity = severity;
+ } else {
+ var index = severity['index'];
+ if (index < highestSeverity) {
+ highestSeverity = severity;
+ }
+ }
+
+ if (!localSeverityMap[severity['index']]) {
+ localSeverityMap[severity['index']] = 0;
+ }
+
+ if (!localLeftoverMap[severity['index']]) {
+ localLeftoverMap[severity['index']] = 0;
+ }
+
+ localSeverityMap[severity['index']]++;
+ if (!vuln.FixedBy) {
+ localLeftoverMap[severity['index']]++;
+ }
+ });
+
+ if (!severityMap[highestSeverity['index']]) {
+ severityMap[highestSeverity['index']] = 0;
+ }
+ severityMap[highestSeverity['index']]++;
+
+ var localSeverityBreakdown = [];
+ var localLeftoverBreakdown = [];
+
+ for (var i = 0; i < levels.length; ++i) {
+ var level = levels[i];
+ if (localSeverityMap[level['index']]) {
+ localSeverityBreakdown.push({
+ 'title': level['title'],
+ 'color': level['color'],
+ 'count': localSeverityMap[level['index']]
+ })
+ }
+
+ if (localLeftoverMap[level['index']]) {
+ localLeftoverBreakdown.push({
+ 'title': level['title'],
+ 'color': level['color'],
+ 'count': localLeftoverMap[level['index']]
+ })
+ }
+ }
+
+ feature_obj['vulnCount'] = feature.Vulnerabilities.length;
+
+ feature_obj['severityBreakdown'] = localSeverityBreakdown;
+ feature_obj['leftoverBreakdown'] = localLeftoverBreakdown;
+
+ if (localSeverityBreakdown) {
+ feature_obj['primarySeverity'] = localSeverityBreakdown[0];
+ }
+
+ if (localLeftoverBreakdown) {
+ feature_obj['primaryLeftover'] = localLeftoverBreakdown[0];
+ }
+
+ if (feature.Vulnerabilities.length > 0) {
+ feature_obj['score'] = feature_obj['score'] / feature.Vulnerabilities.length;
+ }
+
+ if (feature_obj['fixableScore'] > $scope.highestFixableScore) {
+ $scope.highestFixableScore = feature_obj['fixableScore'];
+ }
+ } else {
+ feature_obj['fixableScore'] = -1;
+ }
+
+ $scope.securityFeatures.push(feature_obj);
+ });
+ }
+
+ var greenCount = $scope.securityFeatures.length;
+ for (var i = 0; i < levels.length; ++i) {
+ var level = levels[i];
+ if (!severityMap[level['index']]) {
+ continue
+ }
+
+ greenCount -= severityMap[level['index']];
+ $scope.featureBreakdown.push({
+ 'label': levels[i].title,
+ 'value': severityMap[level['index']],
+ 'color': levels[i].color,
+ });
+ }
+
+ $scope.featureBreakdown.push({
+ 'label': 'None',
+ 'value': greenCount,
+ 'color': '#2FC98E'
+ });
+
+ buildOrderedFeatures();
+ };
+
+ var loadImageVulnerabilities = function() {
+ if ($scope.securityResource) {
+ return;
+ }
+
+ var params = {
+ 'repository': $scope.repository.namespace + '/' + $scope.repository.name,
+ 'imageid': $scope.image.id,
+ 'vulnerabilities': true,
+ };
+
+ $scope.securityResource = ApiService.getRepoImageSecurityAsResource(params).get(function(resp) {
+ $scope.securityStatus = resp.status;
+ buildFeatures(resp.data);
+ buildChart();
+ return resp;
+ });
+ };
+
+ $scope.$watch('options.predicate', buildOrderedFeatures);
+ $scope.$watch('options.reverse', buildOrderedFeatures);
+ $scope.$watch('options.featureFilter', buildOrderedFeatures);
+
+ $scope.$watch('isEnabled', function(isEnabled) {
+ if ($scope.isEnabled && $scope.repository && $scope.image) {
+ loadImageVulnerabilities();
+ }
+ });
+ }
+ };
+ return directiveDefinitionObject;
+});
\ No newline at end of file
diff --git a/static/js/directives/ui/image-vulnerability-view.js b/static/js/directives/ui/image-vulnerability-view.js
new file mode 100644
index 000000000..2932b8c4e
--- /dev/null
+++ b/static/js/directives/ui/image-vulnerability-view.js
@@ -0,0 +1,252 @@
+/**
+ * An element which displays the vulnerabilities in an image.
+ */
+angular.module('quay').directive('imageVulnerabilityView', function () {
+ var directiveDefinitionObject = {
+ priority: 0,
+ templateUrl: '/static/directives/image-vulnerability-view.html',
+ replace: false,
+ transclude: true,
+ restrict: 'C',
+ scope: {
+ 'repository': '=repository',
+ 'image': '=image',
+ 'isEnabled': '=isEnabled'
+ },
+ controller: function($scope, $element, Config, ApiService, VulnerabilityService, AngularViewArray, ImageMetadataService) {
+ var imageMap = null;
+
+ $scope.securityVulnerabilities = [];
+
+ $scope.options = {
+ 'vulnFilter': null,
+ 'fixableVulns': false,
+ 'predicate': 'score',
+ 'reverse': false,
+ };
+
+ $scope.tablePredicateClass = function(name, predicate, reverse) {
+ if (name != predicate) {
+ return '';
+ }
+
+ return 'current ' + (reverse ? 'reversed' : '');
+ };
+
+ $scope.orderBy = function(predicate) {
+ if (predicate == $scope.options.predicate) {
+ $scope.options.reverse = !$scope.options.reverse;
+ return;
+ }
+
+ $scope.options.reverse = false;
+ $scope.options.predicate = predicate;
+ };
+
+ $scope.getCVSSColor = function(score) {
+ return VulnerabilityService.getCVSSColor(score);
+ };
+
+ $scope.toggleDetails = function(vuln) {
+ vuln.expanded = !vuln.expanded;
+ };
+
+ var buildOrderedVulnerabilities = function() {
+ var vulnerabilities = $scope.securityVulnerabilities.slice(0);
+
+ $scope.orderedVulnerabilities = AngularViewArray.create();
+ vulnerabilities.forEach(function(v) {
+ var vulnFilter = $scope.options.vulnFilter;
+ if (vulnFilter) {
+ if ((v['name'].indexOf(vulnFilter) < 0) &&
+ (v['featureName'].indexOf(vulnFilter) < 0) &&
+ (v['imageCommand'].indexOf(vulnFilter) < 0)) {
+ return;
+ }
+ }
+
+ if ($scope.options.fixableVulns && !v['fixedInVersion']) {
+ return;
+ }
+
+ $scope.orderedVulnerabilities.push(v);
+ });
+
+ $scope.orderedVulnerabilities.entries.sort(function(a, b) {
+ var left = a[$scope.options['predicate']];
+ var right = b[$scope.options['predicate']];
+
+ if ($scope.options['predicate'] == 'score') {
+ left = left * 1;
+ right = right * 1;
+ }
+
+ if (left == null) {
+ left = '0.00';
+ }
+
+ if (right == null) {
+ right = '0.00';
+ }
+
+ if (left == right) {
+ return 0;
+ }
+
+ return left > right ? -1 : 1;
+ });
+
+ if ($scope.options['reverse']) {
+ $scope.orderedVulnerabilities.entries.reverse();
+ }
+
+ $scope.orderedVulnerabilities.setVisible(true);
+ };
+
+ var buildChart = function() {
+ var chartData = $scope.priorityBreakdown;
+ if ($scope.priorityBreakdown.length == 0) {
+ chartData = [{
+ 'label': 'None',
+ 'value': 1,
+ 'color': '#2FC98E'
+ }];
+ }
+
+ var colors = [];
+ for (var i = 0; i < chartData.length; ++i) {
+ colors.push(chartData[i].color);
+ }
+
+ nv.addGraph(function() {
+ var chart = nv.models.pieChart()
+ .x(function(d) { return d.label })
+ .y(function(d) { return d.value })
+ .margin({left: -10, right: -10, top: -10, bottom: -10})
+ .showLegend(false)
+ .showLabels(true)
+ .labelThreshold(.05)
+ .labelType("percent")
+ .donut(true)
+ .color(colors)
+ .donutRatio(0.5);
+
+ d3.select("#vulnDonutChart svg")
+ .datum(chartData)
+ .transition()
+ .duration(350)
+ .call(chart);
+
+ return chart;
+ });
+ };
+
+ var buildFeaturesAndVulns = function(data) {
+ $scope.securityFeatures = [];
+ $scope.securityVulnerabilities = [];
+ $scope.priorityBreakdown = [];
+
+ var severityMap = {};
+
+ if (data && data.Layer && data.Layer.Features) {
+ data.Layer.Features.forEach(function(feature) {
+ feature_obj = {
+ 'name': feature.Name,
+ 'namespace': feature.Namespace,
+ 'version': feature.Version,
+ 'addedBy': feature.AddedBy
+ }
+
+ feature_vulnerabilities = []
+
+ if (feature.Vulnerabilities) {
+ feature.Vulnerabilities.forEach(function(vuln) {
+ var score = VulnerabilityService.LEVELS[vuln['Severity']]['score'];
+ if (vuln.Metadata && vuln.Metadata.NVD && vuln.Metadata.NVD.CVSSv2 && vuln.Metadata.NVD.CVSSv2.Score) {
+ score = vuln.Metadata.NVD.CVSSv2.Score;
+ }
+
+ var imageId = feature.AddedBy.split('.')[0];
+
+ vuln_obj = {
+ 'name': vuln.Name,
+ 'namespace': vuln.Namespace,
+ 'description': vuln.Description,
+ 'link': vuln.Link,
+ 'severity': vuln.Severity,
+ 'metadata': vuln.Metadata,
+ 'feature': jQuery.extend({}, feature_obj),
+
+ 'featureName': feature.Name,
+ 'fixedInVersion': vuln.FixedBy,
+ 'introducedInVersion': feature.Version,
+ 'imageId': imageId,
+ 'imageCommand': ImageMetadataService.getImageCommand($scope.image, imageId),
+ 'score': score,
+
+ 'expanded': false,
+ }
+
+ feature_vulnerabilities.push(vuln_obj)
+ $scope.securityVulnerabilities.push(vuln_obj);
+
+ if (severityMap[vuln.Severity] == undefined) {
+ severityMap[vuln.Severity] = 0;
+ }
+
+ severityMap[vuln.Severity]++;
+ });
+ }
+
+ feature_obj['vulnerabilities'] = feature_vulnerabilities
+ $scope.securityFeatures.push(feature_obj);
+ });
+
+ var levels = VulnerabilityService.getLevels();
+ for (var i = 0; i < levels.length; ++i) {
+ if (severityMap[levels[i].title]) {
+ $scope.priorityBreakdown.push({
+ 'label': levels[i].title,
+ 'value': severityMap[levels[i].title],
+ 'color': levels[i].color,
+ })
+ }
+ }
+ }
+
+ buildOrderedVulnerabilities();
+ };
+
+ var loadImageVulnerabilities = function() {
+ if ($scope.securityResource) {
+ return;
+ }
+
+ var params = {
+ 'repository': $scope.repository.namespace + '/' + $scope.repository.name,
+ 'imageid': $scope.image.id,
+ 'vulnerabilities': true,
+ };
+
+ $scope.securityResource = ApiService.getRepoImageSecurityAsResource(params).get(function(resp) {
+ $scope.securityStatus = resp.status;
+ buildFeaturesAndVulns(resp.data);
+ buildChart();
+ return resp;
+ });
+ };
+
+ $scope.$watch('options.predicate', buildOrderedVulnerabilities);
+ $scope.$watch('options.reverse', buildOrderedVulnerabilities);
+ $scope.$watch('options.vulnFilter', buildOrderedVulnerabilities);
+ $scope.$watch('options.fixableVulns', buildOrderedVulnerabilities);
+
+ $scope.$watch('isEnabled', function(isEnabled) {
+ if ($scope.isEnabled && $scope.repository && $scope.image) {
+ loadImageVulnerabilities();
+ }
+ });
+ }
+ };
+ return directiveDefinitionObject;
+});
\ No newline at end of file
diff --git a/static/js/directives/ui/nvd-vectors-display.js b/static/js/directives/ui/nvd-vectors-display.js
new file mode 100644
index 000000000..c23f3bfcd
--- /dev/null
+++ b/static/js/directives/ui/nvd-vectors-display.js
@@ -0,0 +1,35 @@
+/**
+ * An element which displays NVD vectors is an expanded format.
+ */
+angular.module('quay').directive('nvdVectorsDisplay', function () {
+ var directiveDefinitionObject = {
+ priority: 0,
+ templateUrl: '/static/directives/nvd-vectors-display.html',
+ replace: false,
+ transclude: true,
+ restrict: 'C',
+ scope: {
+ 'vectors': '@vectors',
+ },
+ controller: function($scope, $element, VulnerabilityService) {
+ $scope.parsedVectors = VulnerabilityService.parseVectorsString($scope.vectors);
+
+ $scope.getVectorTitle = function(vector) {
+ return VulnerabilityService.getVectorTitle(vector);
+ };
+
+ $scope.getVectorDescription = function(vector) {
+ return VulnerabilityService.getVectorDescription(vector);
+ };
+
+ $scope.getVectorOptions = function(vectorString) {
+ return VulnerabilityService.getVectorOptions(vectorString);
+ };
+
+ $scope.getVectorClasses = function(option, vectorString) {
+ return VulnerabilityService.getVectorClasses(option, vectorString);
+ };
+ }
+ };
+ return directiveDefinitionObject;
+});
\ No newline at end of file
diff --git a/static/js/pages/image-view.js b/static/js/pages/image-view.js
index 5e377eac0..5db5f31d5 100644
--- a/static/js/pages/image-view.js
+++ b/static/js/pages/image-view.js
@@ -10,14 +10,15 @@
})
}]);
- function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService, VulnerabilityService, Features) {
+ function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService, Features) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
var imageid = $routeParams.image;
+ $scope.imageSecurityCounter = 0;
+ $scope.imagePackageCounter = 0;
$scope.options = {
- 'vulnFilter': '',
- 'packageFilter': ''
+ 'vulnFilter': ''
};
var loadImage = function() {
@@ -46,57 +47,13 @@
loadRepository();
$scope.loadImageSecurity = function() {
- if (!Features.SECURITY_SCANNER || $scope.securityResource) { return; }
+ if (!Features.SECURITY_SCANNER) { return; }
+ $scope.imageSecurityCounter++;
+ };
- $scope.VulnerabilityLevels = VulnerabilityService.getLevels();
-
- var params = {
- 'repository': namespace + '/' + name,
- 'imageid': imageid,
- 'vulnerabilities': true,
- };
-
- $scope.securityResource = ApiService.getRepoImageSecurityAsResource(params).get(function(resp) {
- $scope.securityStatus = resp.status;
- $scope.securityFeatures = [];
- $scope.securityVulnerabilities = [];
-
- if (resp.data && resp.data.Layer && resp.data.Layer.Features) {
- resp.data.Layer.Features.forEach(function(feature) {
- feature_obj = {
- 'name': feature.Name,
- 'namespace': feature.Namespace,
- 'version': feature.Version,
- 'addedBy': feature.AddedBy,
- }
- feature_vulnerabilities = []
-
- if (feature.Vulnerabilities) {
- feature.Vulnerabilities.forEach(function(vuln) {
- vuln_obj = {
- 'name': vuln.Name,
- 'namespace': vuln.Namespace,
- 'description': vuln.Description,
- 'link': vuln.Link,
- 'severity': vuln.Severity,
- 'metadata': vuln.Metadata,
- 'feature': jQuery.extend({}, feature_obj),
- 'fixedBy': vuln.FixedBy,
- 'index': VulnerabilityService.LEVELS[vuln['Severity']]['index'],
- }
-
- feature_vulnerabilities.push(vuln_obj)
- $scope.securityVulnerabilities.push(vuln_obj);
- });
- }
-
- feature_obj['vulnerabilities'] = feature_vulnerabilities
- $scope.securityFeatures.push(feature_obj);
- });
- }
-
- return resp;
- });
+ $scope.loadImagePackages = function() {
+ if (!Features.SECURITY_SCANNER) { return; }
+ $scope.imagePackageCounter++;
};
$scope.initializeTree = function() {
diff --git a/static/js/services/image-metadata-service.js b/static/js/services/image-metadata-service.js
index 819d12a2b..f8a1e1a33 100644
--- a/static/js/services/image-metadata-service.js
+++ b/static/js/services/image-metadata-service.js
@@ -24,5 +24,34 @@ angular.module('quay').factory('ImageMetadataService', ['UtilService', function(
return UtilService.textToSafeHtml(metadataService.getFormattedCommand(image));
};
+ metadataService.getImageCommand = function(image, imageId) {
+ if (!image.__imageMap) {
+ image.__imageMap = {};
+ for (var i = 0; i < image.history.length; ++i) {
+ var cimage = image.history[i];
+ image.__imageMap[cimage.id] = cimage;
+ }
+ }
+
+ return getDockerfileCommand(image.__imageMap[imageId].command);
+ };
+
+ var getDockerfileCommand = function(command) {
+ if (!command) { return ''; }
+
+ // ["/bin/sh", "-c", "#(nop) RUN foo"]
+ var commandPrefix = '#(nop)'
+
+ if (command.length != 3) { return ''; }
+ if (command[0] != '/bin/sh' || command[1] != '-c') { return ''; }
+
+ var cmd = command[2];
+ if (cmd.substring(0, commandPrefix.length) != commandPrefix) {
+ return 'RUN ' + cmd;
+ }
+
+ return command[2].substr(commandPrefix.length + 1);
+ };
+
return metadataService;
}]);
\ No newline at end of file
diff --git a/static/js/services/vulnerability-service.js b/static/js/services/vulnerability-service.js
index 12c1a172f..b6cf51f6f 100644
--- a/static/js/services/vulnerability-service.js
+++ b/static/js/services/vulnerability-service.js
@@ -5,10 +5,234 @@ angular.module('quay').factory('VulnerabilityService', ['Config', function(Confi
var vulnService = {};
vulnService.LEVELS = window.__vuln_priority;
+ var cvssSeverityMap = {};
+
+ vulnService.getSeverityForCVSS = function(score) {
+ if (cvssSeverityMap[score]) {
+ return cvssSeverityMap[score];
+ }
+
+ var levels = vulnService.getLevels();
+ for (var i = 0; i < levels.length; ++i) {
+ if (score >= levels[i].score) {
+ cvssSeverityMap[score] = levels[i];
+ return levels[i];
+ }
+ }
+
+ return vulnService.LEVELS['Unknown'];
+ };
+
+ vulnService.getCVSSColor = function(score) {
+ return vulnService.getSeverityForCVSS(score).color;
+ };
+
vulnService.getLevels = function() {
- return Object.keys(vulnService.LEVELS).map(function(key) {
+ var levels = Object.keys(vulnService.LEVELS).map(function(key) {
return vulnService.LEVELS[key];
});
+ return levels.sort(function(a, b) {
+ return a.index - b.index;
+ });
+ };
+
+ vulnService.parseVectorsString = function(vectorsString) {
+ return vectorsString.split('/');
+ };
+
+ vulnService.getVectorTitle = function(vectorString) {
+ var parts = vectorString.split(':');
+ var vector = vulnService.NVD_VECTORS[parts[0]];
+ if (!vector) {
+ return '';
+ }
+
+ return vector.title;
+ };
+
+ vulnService.getVectorDescription = function(vectorString) {
+ var parts = vectorString.split(':');
+ var vector = vulnService.NVD_VECTORS[parts[0]];
+ if (!vector) {
+ return '';
+ }
+
+ return vector.description;
+ };
+
+ vulnService.getVectorClasses = function(option, vectorString) {
+ var parts = vectorString.split(':');
+ var vector = vulnService.NVD_VECTORS[parts[0]];
+ if (!vector) {
+ return '';
+ }
+
+ var classes = '';
+ if (option.id == parts[1]) {
+ classes += 'current-vector ';
+ } else {
+ classes += 'not-current-vector ';
+ }
+
+ classes += option.severity;
+ return classes
+ };
+
+ vulnService.getVectorOptions = function(vectorString) {
+ var parts = vectorString.split(':');
+ return vulnService.NVD_VECTORS[parts[0]].values;
+ };
+
+ vulnService.NVD_VECTORS = {
+ 'AV': {
+ 'title': 'Access Vector',
+ 'description': 'This metric reflects how the vulnerability is exploited. The more remote an attacker can be to attack a host, the greater the vulnerability score.',
+ 'values': [
+ {
+ 'id': 'N',
+ 'title': 'Network',
+ 'description': 'A vulnerability exploitable with network access means the vulnerable software is bound to the network stack and the attacker does not require local network access or local access. Such a vulnerability is often termed "remotely exploitable". An example of a network attack is an RPC buffer overflow.',
+ 'severity': 'high'
+ },
+ {
+ 'id': 'A',
+ 'title': 'Adjacent Network',
+ 'description': 'A vulnerability exploitable with adjacent network access requires the attacker to have access to either the broadcast or collision domain of the vulnerable software. Examples of local networks include local IP subnet, Bluetooth, IEEE 802.11, and local Ethernet segment.',
+ 'severity': 'medium'
+ },
+ {
+ 'id': 'L',
+ 'title': 'Local',
+ 'description': 'A vulnerability exploitable with only local access requires the attacker to have either physical access to the vulnerable system or a local (shell) account. Examples of locally exploitable vulnerabilities are peripheral attacks such as Firewire/USB DMA attacks, and local privilege escalations (e.g., sudo).',
+ 'severity': 'low'
+ }
+ ]
+ },
+
+ 'AC': {
+ 'title': 'Access Complexity',
+ 'description': 'This metric measures the complexity of the attack required to exploit the vulnerability once an attacker has gained access to the target system. For example, consider a buffer overflow in an Internet service: once the target system is located, the attacker can launch an exploit at will.',
+ 'values': [
+ {
+ 'id': 'L',
+ 'title': 'Low',
+ 'description': 'Specialized access conditions or extenuating circumstances do not exist making this easy to exploit',
+ 'severity': 'high'
+ },
+ {
+ 'id': 'M',
+ 'title': 'Medium',
+ 'description': 'The access conditions are somewhat specialized making this somewhat difficult to exploit',
+ 'severity': 'medium'
+ },
+ {
+ 'id': 'H',
+ 'title': 'High',
+ 'description': 'Specialized access conditions exist making this harder to exploit',
+ 'severity': 'low'
+ }
+ ]
+ },
+
+ 'Au': {
+ 'title': 'Authentication',
+ 'description': 'This metric measures the number of times an attacker must authenticate to a target in order to exploit a vulnerability. This metric does not gauge the strength or complexity of the authentication process, only that an attacker is required to provide credentials before an exploit may occur. The fewer authentication instances that are required, the higher the vulnerability score.',
+ 'values': [
+ {
+ 'id': 'N',
+ 'title': 'None',
+ 'description': 'Authentication is not required to exploit the vulnerability.',
+ 'severity': 'high'
+ },
+ {
+ 'id': 'S',
+ 'title': 'Single',
+ 'description': 'The vulnerability requires an attacker to be logged into the system (such as at a command line or via a desktop session or web interface).',
+ 'severity': 'medium'
+ },
+ {
+ 'id': 'M',
+ 'title': 'Multiple',
+ 'description': 'Exploiting the vulnerability requires that the attacker authenticate two or more times, even if the same credentials are used each time. An example is an attacker authenticating to an operating system in addition to providing credentials to access an application hosted on that system.',
+ 'severity': 'low'
+ }
+ ]
+ },
+
+ 'C': {
+ 'title': 'Confidentiality Impact',
+ 'description': 'This metric measures the impact on confidentiality of a successfully exploited vulnerability. Confidentiality refers to limiting information access and disclosure to only authorized users, as well as preventing access by, or disclosure to, unauthorized ones. Increased confidentiality impact increases the vulnerability score.',
+ 'values': [
+ {
+ 'id': 'C',
+ 'title': 'Complete',
+ 'description': 'There is total information disclosure, resulting in all system files being revealed. The attacker is able to read all of the system\'s data (memory, files, etc.)',
+ 'severity': 'high'
+ },
+ {
+ 'id': 'P',
+ 'title': 'Partial',
+ 'description': 'There is considerable informational disclosure. Access to some system files is possible, but the attacker does not have control over what is obtained, or the scope of the loss is constrained. An example is a vulnerability that divulges only certain tables in a database.',
+ 'severity': 'medium'
+ },
+ {
+ 'id': 'N',
+ 'title': 'None',
+ 'description': 'There is no impact to the confidentiality of the system.',
+ 'severity': 'low'
+ }
+ ]
+ },
+
+ 'I': {
+ 'title': 'Integrity Impact',
+ 'description': 'This metric measures the impact to integrity of a successfully exploited vulnerability. Integrity refers to the trustworthiness and guaranteed veracity of information. Increased integrity impact increases the vulnerability score.',
+ 'values': [
+ {
+ 'id': 'C',
+ 'title': 'Complete',
+ 'description': 'There is a total compromise of system integrity. There is a complete loss of system protection, resulting in the entire system being compromised. The attacker is able to modify any files on the target system',
+ 'severity': 'high'
+ },
+ {
+ 'id': 'P',
+ 'title': 'Partial',
+ 'description': 'Modification of some system files or information is possible, but the attacker does not have control over what can be modified, or the scope of what the attacker can affect is limited. For example, system or application files may be overwritten or modified, but either the attacker has no control over which files are affected or the attacker can modify files within only a limited context or scope.',
+ 'severity': 'medium'
+ },
+ {
+ 'id': 'N',
+ 'title': 'None',
+ 'description': 'There is no impact to the integrity of the system.',
+ 'severity': 'low'
+ }
+ ]
+ },
+
+ 'A': {
+ 'title': 'Availability Impact',
+ 'description': 'This metric measures the impact to availability of a successfully exploited vulnerability. Availability refers to the accessibility of information resources. Attacks that consume network bandwidth, processor cycles, or disk space all impact the availability of a system. Increased availability impact increases the vulnerability score.',
+ 'values': [
+ {
+ 'id': 'C',
+ 'title': 'Complete',
+ 'description': 'There is a total shutdown of the affected resource. The attacker can render the resource completely unavailable.',
+ 'severity': 'high'
+ },
+ {
+ 'id': 'P',
+ 'title': 'Partial',
+ 'description': 'There is reduced performance or interruptions in resource availability. An example is a network-based flood attack that permits a limited number of successful connections to an Internet service.',
+ 'severity': 'medium'
+ },
+ {
+ 'id': 'N',
+ 'title': 'None',
+ 'description': 'There is no impact to the availability of the system.',
+ 'severity': 'low'
+ }
+ ]
+ }
};
return vulnService;
diff --git a/static/partials/image-view.html b/static/partials/image-view.html
index b3343802f..47b99cd4e 100644
--- a/static/partials/image-view.html
+++ b/static/partials/image-view.html
@@ -27,7 +27,7 @@
<i class="fa fa-bug"></i>
</span>
<span class="cor-tab" tab-title="Packages" tab-target="#packages"
- tab-init="loadImageSecurity()"
+ tab-init="loadImagePackages()"
quay-show="Features.SECURITY_SCANNER">
<i class="fa ci-package"></i>
</span>
@@ -44,123 +44,12 @@
<!-- Vulnerabilities -->
<div id="vulnerabilities" class="tab-pane" quay-require="['SECURITY_SCANNER']">
- <div class="resource-view" resource="securityResource" error-message="'Could not load security information for image'">
- <div class="col-md-9">
- <div class="filter-box floating" collection="securityVulnerabilities" filter-model="options.vulnFilter" filter-name="Vulnerabilities" ng-if="securityStatus == 'scanned' && securityVulnerabilities.length"></div>
-
- <h3>Image Security</h3>
- <div class="empty" ng-if="securityStatus == 'queued'">
- <div class="empty-primary-msg">This image has not been indexed yet</div>
- <div class="empty-secondary-msg">
- Please try again in a few minutes.
- </div>
- </div>
-
- <div class="empty" ng-if="securityStatus == 'failed'">
- <div class="empty-primary-msg">This image could not be indexed</div>
- <div class="empty-secondary-msg">
- Our security scanner was unable to index this image.
- </div>
- </div>
-
- <div class="empty" ng-if="securityStatus == 'scanned' && !securityVulnerabilities.length">
- <div class="empty-primary-msg">This image contains no recognized security vulnerabilities</div>
- <div class="empty-secondary-msg">
- Quay currently indexes Debian, Red Hat and Ubuntu based images.
- </div>
- </div>
-
- <div ng-if="securityStatus == 'scanned' && securityVulnerabilities.length">
- <table class="co-table">
- <thead>
- <td style="width: 200px;">Vulnerability</td>
- <td style="width: 200px;">Priority</td>
- <td style="width: 200px;">Introduced by</td>
- <td style="width: 200px;">Fixed by</td>
- <td>Description</td>
- </thead>
-
- <tr ng-repeat="vulnerability in securityVulnerabilities | filter:options.vulnFilter | orderBy:'index'">
- <td><a href="{{ vulnerability.link }}" target="_blank">{{ vulnerability.name }}</a></td>
- <td>
- <span class="vulnerability-priority-view" priority="vulnerability.severity"></span>
- </td>
- <td>{{ vulnerability.feature.name }} {{ vulnerability.feature.version }}</td>
- <td>
- <span ng-if="vulnerability.fixedBy">{{ vulnerability.feature.name }} {{ vulnerability.fixedBy }}</span>
- </td>
- <td>{{ vulnerability.description }}</td>
- </tr>
- </table>
-
- <div class="empty" ng-if="(securityVulnerabilities | filter:options.vulnFilter).length == 0"
- style="margin-top: 20px;">
- <div class="empty-primary-msg">No matching vulnerabilities found</div>
- <div class="empty-secondary-msg">
- Please adjust your filter above.
- </div>
- </div>
- </div>
- </div>
-
- <div class="level-col col-md-3 hidden-sm hidden-xs">
- <h4>Priority Guide</h4>
- <ul class="levels">
- <li ng-repeat="level in VulnerabilityLevels | orderBy: 'index'">
- <div class="vulnerability-priority-view" priority="level.title"></div>
- <div class="description">
- {{ level.description }}
- </div>
- </li>
- </ul>
- </div>
- </div>
+ <div class="image-vulnerability-view" repository="repository" image="image" is-enabled="imageSecurityCounter"></div>
</div>
<!-- Features -->
<div id="packages" class="tab-pane" quay-require="['SECURITY_SCANNER']">
- <div class="resource-view" resource="securityResource" error-message="'Could not load image packages'">
- <div class="filter-box floating" collection="securityFeatures" filter-model="options.packageFilter" filter-name="Features" ng-if="securityStatus == 'scanned' && securityFeatures.length"></div>
-
- <h3>Image Packages</h3>
- <div class="empty" ng-if="securityStatus == 'queued'">
- <div class="empty-primary-msg">This image has not been indexed yet</div>
- <div class="empty-secondary-msg">
- Please try again in a few minutes.
- </div>
- </div>
-
- <div class="empty" ng-if="securityStatus == 'failed'">
- <div class="empty-primary-msg">This image could not be indexed</div>
- <div class="empty-secondary-msg">
- Our security scanner was unable to index this image.
- </div>
- </div>
-
- <table class="co-table" ng-if="securityStatus == 'scanned'">
- <thead>
- <td>Package Name</td>
- <td>Package Version</td>
- <td>Package OS</td>
- <td>Number of vulnerabilities</td>
- </thead>
-
- <tr ng-repeat="feature in securityFeatures | filter:options.packageFilter | orderBy:'Name'">
- <td>{{ feature.name }}</td>
- <td>{{ feature.version }}</td>
- <td>{{ feature.namespace }}</td>
- <td>{{ feature.vulnerabilities.length }}</td>
- </tr>
- </table>
-
- <div class="empty" ng-if="(securityFeatures | filter:options.packageFilter).length == 0"
- style="margin-top: 20px;">
- <div class="empty-primary-msg">No matching packages found</div>
- <div class="empty-secondary-msg" ng-if="options.packageFilter">
- Please adjust your filter above.
- </div>
- </div>
- </div>
+ <div class="image-feature-view" repository="repository" image="image" is-enabled="imagePackageCounter"></div>
</div>
</div>
</div>
diff --git a/util/secscan/__init__.py b/util/secscan/__init__.py
index 1588e2a16..c19f93825 100644
--- a/util/secscan/__init__.py
+++ b/util/secscan/__init__.py
@@ -5,6 +5,8 @@ PRIORITY_LEVELS = {
'title': 'Unknown',
'index': 6,
'level': 'info',
+ 'color': '#9B9B9B',
+ 'score': 0,
'description': 'Unknown is either a security problem that has not been assigned ' +
'to a priority yet or a priority that our system did not recognize',
@@ -15,6 +17,8 @@ PRIORITY_LEVELS = {
'title': 'Negligible',
'index': 5,
'level': 'info',
+ 'color': '#9B9B9B',
+ 'score': 1,
'description': 'Negligible is technically a security problem, but is only theoretical ' +
'in nature, requires a very special situation, has almost no install base, ' +
@@ -26,6 +30,8 @@ PRIORITY_LEVELS = {
'title': 'Low',
'index': 4,
'level': 'warning',
+ 'color': 'rgb(204, 201, 46)',
+ 'score': 3,
'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
'requires a user-assisted attack, a small install base, or does very ' +
@@ -38,6 +44,8 @@ PRIORITY_LEVELS = {
'value': 'Medium',
'index': 3,
'level': 'warning',
+ 'color': 'rgb(252, 166, 87)',
+ 'score': 6,
'description': 'Medium is a real security problem, and is exploitable for many people. ' +
'Includes network daemon denial of service attacks, cross-site scripting, ' +
@@ -50,6 +58,8 @@ PRIORITY_LEVELS = {
'value': 'High',
'index': 2,
'level': 'warning',
+ 'color': '#D64456',
+ 'score': 9,
'description': 'High is a real problem, exploitable for many people in a default installation. ' +
'Includes serious remote denial of services, local root privilege escalations, ' +
@@ -62,6 +72,8 @@ PRIORITY_LEVELS = {
'value': 'Critical',
'index': 1,
'level': 'error',
+ 'color': '#D64456',
+ 'score': 10,
'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
'a installation of the package. Includes remote root privilege escalations, ' +
@@ -74,6 +86,8 @@ PRIORITY_LEVELS = {
'value': 'Defcon1',
'index': 0,
'level': 'error',
+ 'color': 'black',
+ 'score': 11,
'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +
'by the Quay team. It requires immediate attention.',
From 58ccda65904ff276c3c0a0c988e6ba07f1f51f64 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <josephschorr@users.noreply.github.com>
Date: Thu, 25 Feb 2016 17:03:15 -0500
Subject: [PATCH 09/12] Small JS fixes
---
.../directives/image-vulnerability-view.html | 2 +-
.../directives/ui/image-vulnerability-view.js | 20 ++++++++++---------
static/js/services/image-metadata-service.js | 2 +-
static/js/services/vulnerability-service.js | 2 +-
4 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/static/directives/image-vulnerability-view.html b/static/directives/image-vulnerability-view.html
index 919dd1966..321289018 100644
--- a/static/directives/image-vulnerability-view.html
+++ b/static/directives/image-vulnerability-view.html
@@ -105,7 +105,7 @@
<span class="fixed-in-version" bo-if="vuln.fixedInVersion" bo-text="vuln.fixedInVersion"></span>
</td>
<td class="double-col image-col hidden-xs hidden-sm hidden-md">
- <span data-title="{{ vuln.imageCommand }}" bs-tooltip>
+ <span data-title="{{ vuln.imageCommand }}" data-container="body" bs-tooltip>
<span class="dockerfile-command" command="vuln.imageCommand"></span>
</span>
<a href="/repository/{{ repository.namespace }}/{{ repository.name }}/image/{{ vuln.imageId }}"><i class="fa fa-archive"></i></a>
diff --git a/static/js/directives/ui/image-vulnerability-view.js b/static/js/directives/ui/image-vulnerability-view.js
index 2932b8c4e..991b487ee 100644
--- a/static/js/directives/ui/image-vulnerability-view.js
+++ b/static/js/directives/ui/image-vulnerability-view.js
@@ -157,13 +157,15 @@ angular.module('quay').directive('imageVulnerabilityView', function () {
'addedBy': feature.AddedBy
}
- feature_vulnerabilities = []
+ feature_vulnerabilities = [];
if (feature.Vulnerabilities) {
feature.Vulnerabilities.forEach(function(vuln) {
- var score = VulnerabilityService.LEVELS[vuln['Severity']]['score'];
+ var severity = VulnerabilityService.LEVELS[vuln['Severity']];
+ var score = severity.score;
if (vuln.Metadata && vuln.Metadata.NVD && vuln.Metadata.NVD.CVSSv2 && vuln.Metadata.NVD.CVSSv2.Score) {
score = vuln.Metadata.NVD.CVSSv2.Score;
+ severity = VulnerabilityService.getSeverityForCVSS(score);
}
var imageId = feature.AddedBy.split('.')[0];
@@ -190,25 +192,25 @@ angular.module('quay').directive('imageVulnerabilityView', function () {
feature_vulnerabilities.push(vuln_obj)
$scope.securityVulnerabilities.push(vuln_obj);
- if (severityMap[vuln.Severity] == undefined) {
- severityMap[vuln.Severity] = 0;
+ if (severityMap[severity['index']] == undefined) {
+ severityMap[severity['index']] = 0;
}
- severityMap[vuln.Severity]++;
+ severityMap[severity['index']]++;
});
}
- feature_obj['vulnerabilities'] = feature_vulnerabilities
+ feature_obj['vulnerabilities'] = feature_vulnerabilities;
$scope.securityFeatures.push(feature_obj);
});
var levels = VulnerabilityService.getLevels();
for (var i = 0; i < levels.length; ++i) {
- if (severityMap[levels[i].title]) {
+ if (severityMap[levels[i]['index']]) {
$scope.priorityBreakdown.push({
'label': levels[i].title,
- 'value': severityMap[levels[i].title],
- 'color': levels[i].color,
+ 'value': severityMap[levels[i]['index']],
+ 'color': levels[i].color
})
}
}
diff --git a/static/js/services/image-metadata-service.js b/static/js/services/image-metadata-service.js
index f8a1e1a33..e3bf18706 100644
--- a/static/js/services/image-metadata-service.js
+++ b/static/js/services/image-metadata-service.js
@@ -40,7 +40,7 @@ angular.module('quay').factory('ImageMetadataService', ['UtilService', function(
if (!command) { return ''; }
// ["/bin/sh", "-c", "#(nop) RUN foo"]
- var commandPrefix = '#(nop)'
+ var commandPrefix = '#(nop)';
if (command.length != 3) { return ''; }
if (command[0] != '/bin/sh' || command[1] != '-c') { return ''; }
diff --git a/static/js/services/vulnerability-service.js b/static/js/services/vulnerability-service.js
index b6cf51f6f..1a03bdfad 100644
--- a/static/js/services/vulnerability-service.js
+++ b/static/js/services/vulnerability-service.js
@@ -75,7 +75,7 @@ angular.module('quay').factory('VulnerabilityService', ['Config', function(Confi
}
classes += option.severity;
- return classes
+ return classes;
};
vulnService.getVectorOptions = function(vectorString) {
From 4c5c46aa8fcb7adc5cf1e13a7c66de2ad7ffbff8 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <josephschorr@users.noreply.github.com>
Date: Thu, 25 Feb 2016 17:42:09 -0500
Subject: [PATCH 10/12] Add experiment to hide the new Clair UI
---
.../directives/old-image-security-view.html | 116 ++++++++++++++++++
static/js/app.js | 3 +
.../directives/ui/old-image-security-view.js | 84 +++++++++++++
static/js/pages/exp-new-sec-layout.js | 19 +++
static/js/pages/image-view.js | 4 +-
static/partials/exp-new-sec-layout.html | 10 ++
static/partials/image-view.html | 8 +-
7 files changed, 241 insertions(+), 3 deletions(-)
create mode 100644 static/directives/old-image-security-view.html
create mode 100644 static/js/directives/ui/old-image-security-view.js
create mode 100644 static/js/pages/exp-new-sec-layout.js
create mode 100644 static/partials/exp-new-sec-layout.html
diff --git a/static/directives/old-image-security-view.html b/static/directives/old-image-security-view.html
new file mode 100644
index 000000000..6676f76db
--- /dev/null
+++ b/static/directives/old-image-security-view.html
@@ -0,0 +1,116 @@
+<div class="old-image-security-view-element">
+ <!-- Vulnerabilities -->
+ <div ng-show="view == 'vulnerabilities'">
+ <div class="resource-view" resource="securityResource" error-message="'Could not load security information for image'">
+ <div class="col-md-9">
+ <div class="filter-box floating" collection="securityVulnerabilities" filter-model="options.vulnFilter" filter-name="Vulnerabilities" ng-if="securityStatus == 'scanned' && securityVulnerabilities.length"></div>
+
+ <h3>Image Security</h3>
+ <div class="empty" ng-if="securityStatus == 'queued'">
+ <div class="empty-primary-msg">This image has not been indexed yet</div>
+ <div class="empty-secondary-msg">
+ Please try again in a few minutes.
+ </div>
+ </div>
+
+ <div class="empty" ng-if="securityStatus == 'failed'">
+ <div class="empty-primary-msg">This image could not be indexed</div>
+ <div class="empty-secondary-msg">
+ Our security scanner was unable to index this image.
+ </div>
+ </div>
+
+ <div class="empty" ng-if="securityStatus == 'scanned' && !securityVulnerabilities.length">
+ <div class="empty-primary-msg">This image contains no recognized security vulnerabilities</div>
+ <div class="empty-secondary-msg">
+ Quay currently indexes Debian, Red Hat and Ubuntu based images.
+ </div>
+ </div>
+
+ <div ng-if="securityStatus == 'scanned' && securityVulnerabilities.length">
+ <table class="co-table">
+ <thead>
+ <td>Vulnerability</td>
+ <td>Priority</td>
+ <td>Introduced by</td>
+ <td>Description</td>
+ </thead>
+
+ <tr ng-repeat="vulnerability in securityVulnerabilities | filter:options.vulnFilter | orderBy:'index'">
+ <td><a href="{{ vulnerability.link }}" target="_blank">{{ vulnerability.name }}</a></td>
+ <td style="white-space: nowrap;">
+ <span class="vulnerability-priority-view" priority="vulnerability.severity"></span>
+ </td>
+ <td style="white-space: nowrap;">{{ vulnerability.feature.name }} {{ vulnerability.feature.version }}</td>
+ <td>{{ vulnerability.description }}</td>
+ </tr>
+ </table>
+
+ <div class="empty" ng-if="(securityVulnerabilities | filter:options.vulnFilter).length == 0"
+ style="margin-top: 20px;">
+ <div class="empty-primary-msg">No matching vulnerabilities found</div>
+ <div class="empty-secondary-msg">
+ Please adjust your filter above.
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="level-col col-md-3 hidden-sm hidden-xs">
+ <h4>Priority Guide</h4>
+ <ul class="levels">
+ <li ng-repeat="level in VulnerabilityLevels | orderBy: 'index'">
+ <div class="vulnerability-priority-view" priority="level.title"></div>
+ <div class="description">
+ {{ level.description }}
+ </div>
+ </li>
+ </ul>
+ </div>
+ </div>
+ </div>
+
+ <!-- Packages -->
+ <div ng-show="view == 'packages'">
+ <div class="resource-view" resource="securityResource" error-message="'Could not load image packages'">
+ <div class="filter-box floating" collection="securityFeatures" filter-model="options.packageFilter" filter-name="Features" ng-if="securityStatus == 'scanned' && securityFeatures.length"></div>
+
+ <h3>Image Packages</h3>
+ <div class="empty" ng-if="securityStatus == 'queued'">
+ <div class="empty-primary-msg">This image has not been indexed yet</div>
+ <div class="empty-secondary-msg">
+ Please try again in a few minutes.
+ </div>
+ </div>
+
+ <div class="empty" ng-if="securityStatus == 'failed'">
+ <div class="empty-primary-msg">This image could not be indexed</div>
+ <div class="empty-secondary-msg">
+ Our security scanner was unable to index this image.
+ </div>
+ </div>
+
+ <table class="co-table" ng-if="securityStatus == 'scanned'">
+ <thead>
+ <td>Package Name</td>
+ <td>Package Version</td>
+ <td>Package OS</td>
+ </thead>
+
+ <tr ng-repeat="feature in securityFeatures | filter:options.packageFilter | orderBy:'name'">
+ <td>{{ feature.name }}</td>
+ <td>{{ feature.version }}</td>
+ <td>{{ feature.namespace }}</td>
+ </tr>
+ </table>
+
+ <div class="empty" ng-if="(securityFeatures | filter:options.packageFilter).length == 0"
+ style="margin-top: 20px;">
+ <div class="empty-primary-msg">No matching packages found</div>
+ <div class="empty-secondary-msg" ng-if="options.packageFilter">
+ Please adjust your filter above.
+ </div>
+ </div>
+ </div>
+ </div>
+</div>
\ No newline at end of file
diff --git a/static/js/app.js b/static/js/app.js
index f80c7826f..a7798c7e1 100644
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -163,6 +163,9 @@ quayApp.config(['$routeProvider', '$locationProvider', 'pages', function($routeP
// Confirm Invite
.route('/confirminvite', 'confirm-invite')
+ // Experiments
+ .route('/__exp/newseclayout', 'exp-new-sec-layout')
+
// Default: Redirect to the landing page
.otherwise({redirectTo: '/'});
}]);
diff --git a/static/js/directives/ui/old-image-security-view.js b/static/js/directives/ui/old-image-security-view.js
new file mode 100644
index 000000000..218c5f466
--- /dev/null
+++ b/static/js/directives/ui/old-image-security-view.js
@@ -0,0 +1,84 @@
+/**
+ * Old image security view until formally released.
+ */
+angular.module('quay').directive('oldImageSecurityView', function () {
+ var directiveDefinitionObject = {
+ priority: 0,
+ templateUrl: '/static/directives/old-image-security-view.html',
+ replace: false,
+ transclude: true,
+ restrict: 'C',
+ scope: {
+ 'repository': '=repository',
+ 'image': '=image',
+ 'isEnabled': '=isEnabled',
+ 'view': '@view'
+ },
+ controller: function($scope, $element, Config, ApiService, Features, VulnerabilityService, ImageMetadataService) {
+ var loadImageSecurity = function() {
+ if (!Features.SECURITY_SCANNER || $scope.securityResource) { return; }
+
+ $scope.VulnerabilityLevels = VulnerabilityService.getLevels();
+ $scope.options = {
+ 'vulnFilter': '',
+ 'packageFilter': ''
+ };
+
+ var params = {
+ 'repository': $scope.repository.namespace + '/' + $scope.repository.name,
+ 'imageid': $scope.image.id,
+ 'vulnerabilities': true,
+ };
+
+ $scope.securityResource = ApiService.getRepoImageSecurityAsResource(params).get(function(resp) {
+ $scope.securityStatus = resp.status;
+ $scope.securityFeatures = [];
+ $scope.securityVulnerabilities = [];
+
+ if (resp.data && resp.data.Layer && resp.data.Layer.Features) {
+ resp.data.Layer.Features.forEach(function(feature) {
+ feature_obj = {
+ 'name': feature.Name,
+ 'namespace': feature.Namespace,
+ 'version': feature.Version,
+ 'addedBy': feature.AddedBy,
+ }
+ feature_vulnerabilities = []
+
+ if (feature.Vulnerabilities) {
+ feature.Vulnerabilities.forEach(function(vuln) {
+ vuln_obj = {
+ 'name': vuln.Name,
+ 'namespace': vuln.Namespace,
+ 'description': vuln.Description,
+ 'link': vuln.Link,
+ 'severity': vuln.Severity,
+ 'metadata': vuln.Metadata,
+ 'feature': jQuery.extend({}, feature_obj),
+ 'fixedBy': vuln.FixedBy,
+ 'index': VulnerabilityService.LEVELS[vuln['Severity']]['index'],
+ }
+
+ feature_vulnerabilities.push(vuln_obj)
+ $scope.securityVulnerabilities.push(vuln_obj);
+ });
+ }
+
+ feature_obj['vulnerabilities'] = feature_vulnerabilities
+ $scope.securityFeatures.push(feature_obj);
+ });
+ }
+
+ return resp;
+ });
+ };
+
+ $scope.$watch('isEnabled', function(isEnabled) {
+ if ($scope.isEnabled && $scope.repository && $scope.image) {
+ loadImageSecurity();
+ }
+ });
+ }
+ };
+ return directiveDefinitionObject;
+});
\ No newline at end of file
diff --git a/static/js/pages/exp-new-sec-layout.js b/static/js/pages/exp-new-sec-layout.js
new file mode 100644
index 000000000..3df719060
--- /dev/null
+++ b/static/js/pages/exp-new-sec-layout.js
@@ -0,0 +1,19 @@
+(function() {
+ /**
+ * Experiment enable page: New layout
+ */
+ angular.module('quayPages').config(['pages', function(pages) {
+ pages.create('exp-new-sec-layout', 'exp-new-sec-layout.html', ExpCtrl, {
+ 'newLayout': true
+ });
+ }]);
+
+ function ExpCtrl($scope, CookieService) {
+ $scope.isEnabled = CookieService.get('quay.exp-new-sec-layout') == 'true';
+
+ $scope.setEnabled = function(value) {
+ $scope.isEnabled = value;
+ CookieService.putPermanent('quay.exp-new-sec-layout', value.toString());
+ };
+ }
+}());
\ No newline at end of file
diff --git a/static/js/pages/image-view.js b/static/js/pages/image-view.js
index 5db5f31d5..c09e9f31f 100644
--- a/static/js/pages/image-view.js
+++ b/static/js/pages/image-view.js
@@ -10,13 +10,15 @@
})
}]);
- function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService, Features) {
+ function ImageViewCtrl($scope, $routeParams, $rootScope, $timeout, ApiService, ImageMetadataService, Features, CookieService) {
var namespace = $routeParams.namespace;
var name = $routeParams.name;
var imageid = $routeParams.image;
$scope.imageSecurityCounter = 0;
$scope.imagePackageCounter = 0;
+ $scope.newUIExperiment = CookieService.get('quay.exp-new-sec-layout') == 'true';
+
$scope.options = {
'vulnFilter': ''
};
diff --git a/static/partials/exp-new-sec-layout.html b/static/partials/exp-new-sec-layout.html
new file mode 100644
index 000000000..4d19deeb2
--- /dev/null
+++ b/static/partials/exp-new-sec-layout.html
@@ -0,0 +1,10 @@
+<div class="page-content">
+ <div class="cor-title">
+ <span class="cor-title-link"></span>
+ <span class="cor-title-content">Experiment: New Security Scanner Layout</span>
+ </div>
+ <div class="co-main-content-panel">
+ <button class="btn btn-success" ng-if="!isEnabled" ng-click="setEnabled(true)">Enable Experiment</button>
+ <button class="btn btn-failure" ng-if="isEnabled" ng-click="setEnabled(false)">Disable Experiment</button>
+ </div>
+</div>
\ No newline at end of file
diff --git a/static/partials/image-view.html b/static/partials/image-view.html
index 47b99cd4e..2feaf53dc 100644
--- a/static/partials/image-view.html
+++ b/static/partials/image-view.html
@@ -44,12 +44,16 @@
<!-- Vulnerabilities -->
<div id="vulnerabilities" class="tab-pane" quay-require="['SECURITY_SCANNER']">
- <div class="image-vulnerability-view" repository="repository" image="image" is-enabled="imageSecurityCounter"></div>
+ <div class="image-vulnerability-view" repository="repository" image="image" is-enabled="imageSecurityCounter" ng-if="newUIExperiment"></div>
+
+ <div class="old-image-security-view" repository="repository" image="image" is-enabled="imageSecurityCounter" ng-if="!newUIExperiment" view="vulnerabilities"></div>
</div>
<!-- Features -->
<div id="packages" class="tab-pane" quay-require="['SECURITY_SCANNER']">
- <div class="image-feature-view" repository="repository" image="image" is-enabled="imagePackageCounter"></div>
+ <div class="image-feature-view" repository="repository" image="image" is-enabled="imagePackageCounter" ng-if="newUIExperiment"></div>
+
+ <div class="old-image-security-view" repository="repository" image="image" is-enabled="imagePackageCounter" ng-if="!newUIExperiment" view="packages"></div>
</div>
</div>
</div>
From 85919cbc3969a1cb8981c451c50aa040befc6562 Mon Sep 17 00:00:00 2001
From: Joseph Schorr <josephschorr@users.noreply.github.com>
Date: Thu, 25 Feb 2016 17:45:49 -0500
Subject: [PATCH 11/12] Fix error when constructing DownstreamIssue exception
---
endpoints/api/secscan.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/endpoints/api/secscan.py b/endpoints/api/secscan.py
index 47197295f..29c938dec 100644
--- a/endpoints/api/secscan.py
+++ b/endpoints/api/secscan.py
@@ -59,7 +59,7 @@ class RepositoryImageSecurity(RepositoryParamResource):
else:
data = secscan_api.get_layer_data(repo_image, include_features=True)
except APIRequestFailure as arf:
- raise DownstreamIssue(arf.message)
+ raise DownstreamIssue({'message': arf.message})
if data is None:
raise NotFound()
From 672168ce786007892d7c11f10178508fe62ddebd Mon Sep 17 00:00:00 2001
From: Quentin Machu <me@quentin-machu.fr>
Date: Mon, 29 Feb 2016 13:08:49 -0500
Subject: [PATCH 12/12] Close Clair API connections
This forces every API calls to be load-balanced properly.
---
util/secscan/api.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/util/secscan/api.py b/util/secscan/api.py
index 4fc652b06..3e1d72cd0 100644
--- a/util/secscan/api.py
+++ b/util/secscan/api.py
@@ -246,18 +246,19 @@ class SecurityScannerAPI(object):
url = urljoin(api_url, relative_url)
client = self.config['HTTPCLIENT']
+ headers = {'Connection': 'close'}
timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
with CloseForLongOperation(self.config):
if method == 'POST':
logger.debug('POSTing security URL %s', url)
return client.post(url, json=body, params=params, timeout=timeout, cert=self._keys,
- verify=self._certificate)
+ verify=self._certificate, headers=headers)
elif method == 'DELETE':
logger.debug('DELETEing security URL %s', url)
return client.delete(url, params=params, timeout=timeout, cert=self._keys,
- verify=self._certificate)
+ verify=self._certificate, headers=headers)
else:
logger.debug('GETing security URL %s', url)
return client.get(url, params=params, timeout=timeout, cert=self._keys,
- verify=self._certificate)
+ verify=self._certificate, headers=headers)