Implement against new Clair paginated notification system

This commit is contained in:
Joseph Schorr 2016-02-25 15:58:42 -05:00
parent b34314a584
commit f498e92d58
10 changed files with 447 additions and 101 deletions

View file

@ -3,7 +3,7 @@
PRIORITY_LEVELS = {
'Unknown': {
'title': 'Unknown',
'index': '6',
'index': 6,
'level': 'info',
'description': 'Unknown is either a security problem that has not been assigned ' +
@ -13,7 +13,7 @@ PRIORITY_LEVELS = {
'Negligible': {
'title': 'Negligible',
'index': '5',
'index': 5,
'level': 'info',
'description': 'Negligible is technically a security problem, but is only theoretical ' +
@ -24,7 +24,7 @@ PRIORITY_LEVELS = {
'Low': {
'title': 'Low',
'index': '4',
'index': 4,
'level': 'warning',
'description': 'Low is a security problem, but is hard to exploit due to environment, ' +
@ -36,7 +36,7 @@ PRIORITY_LEVELS = {
'Medium': {
'title': 'Medium',
'value': 'Medium',
'index': '3',
'index': 3,
'level': 'warning',
'description': 'Medium is a real security problem, and is exploitable for many people. ' +
@ -48,7 +48,7 @@ PRIORITY_LEVELS = {
'High': {
'title': 'High',
'value': 'High',
'index': '2',
'index': 2,
'level': 'warning',
'description': 'High is a real problem, exploitable for many people in a default installation. ' +
@ -60,7 +60,7 @@ PRIORITY_LEVELS = {
'Critical': {
'title': 'Critical',
'value': 'Critical',
'index': '1',
'index': 1,
'level': 'error',
'description': 'Critical is a world-burning problem, exploitable for nearly all people in ' +
@ -72,7 +72,7 @@ PRIORITY_LEVELS = {
'Defcon1': {
'title': 'Defcon 1',
'value': 'Defcon1',
'index': '0',
'index': 0,
'level': 'error',
'description': 'Defcon1 is a Critical problem which has been manually highlighted ' +

View file

@ -19,8 +19,8 @@ class APIRequestFailure(Exception):
_API_METHOD_INSERT = 'layers'
_API_METHOD_GET_LAYER = 'layers/%s'
_API_METHOD_GET_WITH_VULNERABILITIES_FLAG = '?vulnerabilities'
_API_METHOD_GET_WITH_FEATURES_FLAG = '?features'
_API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s'
_API_METHOD_GET_NOTIFICATION = 'notifications/%s'
class SecurityScannerAPI(object):
@ -113,7 +113,7 @@ class SecurityScannerAPI(object):
logger.info('Analyzing layer %s', request['Layer']['Name'])
try:
response = self._call(_API_METHOD_INSERT, request)
response = self._call('POST', _API_METHOD_INSERT, request)
json_response = response.json()
except requests.exceptions.Timeout:
logger.exception('Timeout when trying to post layer data response for %s', layer.id)
@ -146,35 +146,94 @@ class SecurityScannerAPI(object):
return api_version, False
def check_layer_vulnerable(self, layer_id, cve_name):
""" Checks to see if the layer with the given ID is vulnerable to the specified CVE. """
layer_data = self._get_layer_data(layer_id, include_vulnerabilities=True)
if layer_data is None or 'Layer' not in layer_data or 'Features' not in layer_data['Layer']:
return False
for feature in layer_data['Layer']['Features']:
for vuln in feature.get('Vulnerabilities', []):
if vuln['Name'] == cve_name:
return True
return False
def get_notification(self, notification_name, layer_limit=10, page=None):
""" Gets the data for a specific notification, with optional page token.
Returns a tuple of the data (None on failure) and whether to retry.
"""
try:
params = {
'limit': layer_limit
}
if page is not None:
params['page'] = page
response = self._call('GET', _API_METHOD_GET_NOTIFICATION % notification_name, params=params)
json_response = response.json()
except requests.exceptions.Timeout:
logger.exception('Timeout when trying to get notification for %s', notification_name)
return None, True
except requests.exceptions.ConnectionError:
logger.exception('Connection error when trying to get notification for %s', notification_name)
return None, True
except (requests.exceptions.RequestException, ValueError):
logger.exception('Failed to get notification for %s', notification_name)
return None, False
if response.status_code != 200:
return None, response.status_code != 404 and response.status_code != 400
return json_response, False
def mark_notification_read(self, notification_name):
""" Marks a security scanner notification as read. """
try:
response = self._call('DELETE', _API_METHOD_MARK_NOTIFICATION_READ % notification_name)
return response.status_code == 200
except requests.exceptions.RequestException:
logger.exception('Failed to mark notification as read: %s', notification_name)
return False
def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
""" Returns the layer data for the specified layer. On error, returns None. """
layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
return self._get_layer_data(layer_id, include_features, include_vulnerabilities)
def _get_layer_data(self, layer_id, include_features=False, include_vulnerabilities=False):
try:
flag = ''
params = {}
if include_features:
flag = _API_METHOD_GET_WITH_FEATURES_FLAG
params = {'features': True}
if include_vulnerabilities:
flag = _API_METHOD_GET_WITH_VULNERABILITIES_FLAG
params = {'vulnerabilities': True}
response = self._call(_API_METHOD_GET_LAYER + flag, None, layer_id)
response = self._call('GET', _API_METHOD_GET_LAYER % layer_id, params=params)
logger.debug('Got response %s for vulnerabilities for layer %s',
response.status_code, layer_id)
json_response = response.json()
except requests.exceptions.Timeout:
raise APIRequestFailure('API call timed out')
except requests.exceptions.ConnectionError:
raise APIRequestFailure('Could not connect to security service')
except (requests.exceptions.RequestException, ValueError):
logger.exception('Failed to get layer data response for %s', layer.id)
logger.exception('Failed to get layer data response for %s', layer_id)
raise APIRequestFailure()
if response.status_code == 404:
return None
return response.json()
return json_response
def _call(self, relative_url, body=None, *args, **kwargs):
def _call(self, method, relative_url, params=None, body=None):
""" Issues an HTTP call to the sec API at the given relative URL.
This function disconnects from the database while awaiting a response
from the API server.
@ -184,18 +243,21 @@ class SecurityScannerAPI(object):
raise Exception('Cannot call unconfigured security system')
api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
url = urljoin(api_url, relative_url % args)
url = urljoin(api_url, relative_url)
client = self.config['HTTPCLIENT']
timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
logger.debug('Looking up sec information: %s', url)
with CloseForLongOperation(self.config):
if body is not None:
if method == 'POST':
logger.debug('POSTing security URL %s', url)
return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self._keys,
return client.post(url, json=body, params=params, timeout=timeout, cert=self._keys,
verify=self._certificate)
elif method == 'DELETE':
logger.debug('DELETEing security URL %s', url)
return client.delete(url, params=params, timeout=timeout, cert=self._keys,
verify=self._certificate)
else:
logger.debug('GETing security URL %s', url)
return client.get(url, params=kwargs, timeout=timeout, cert=self._keys,
return client.get(url, params=params, timeout=timeout, cert=self._keys,
verify=self._certificate)

103
util/secscan/notifier.py Normal file
View file

@ -0,0 +1,103 @@
import logging
import sys
from collections import defaultdict
from app import secscan_api
from data.model.tag import filter_tags_have_repository_event, get_matching_tags
from data.database import (Image, ImageStorage, ExternalNotificationEvent, Repository,
RepositoryTag)
from endpoints.notificationhelper import spawn_notification
from util.secscan import PRIORITY_LEVELS
from util.secscan.api import APIRequestFailure
logger = logging.getLogger(__name__)
def process_notification_data(notification_data):
""" Processes the given notification data to spawn vulnerability notifications as necessary.
Returns whether the processing succeeded.
"""
new_data = notification_data['New']
old_data = notification_data.get('Old', {})
new_vuln = new_data['Vulnerability']
old_vuln = old_data.get('Vulnerability', {})
new_layer_ids = set(new_data.get('LayersIntroducingVulnerability', []))
old_layer_ids = set(old_data.get('LayersIntroducingVulnerability', []))
new_severity = PRIORITY_LEVELS.get(new_vuln.get('Severity', 'Unknown'), {'index': sys.maxint})
old_severity = PRIORITY_LEVELS.get(old_vuln.get('Severity', 'Unknown'), {'index': sys.maxint})
# By default we only notify the new layers that are affected by the vulnerability. If, however,
# the severity of the vulnerability has increased, we need to notify *all* layers, as we might
# need to send new notifications for older layers.
notify_layers = new_layer_ids - old_layer_ids
if new_severity['index'] < old_severity['index']:
notify_layers = new_layer_ids | old_layer_ids
if not notify_layers:
# Nothing more to do.
return True
# Lookup the external event for when we have vulnerabilities.
event = ExternalNotificationEvent.get(name='vulnerability_found')
# For each layer, retrieving the matching tags and join with repository to determine which
# require new notifications.
tag_map = defaultdict(set)
repository_map = {}
cve_id = new_vuln['Name']
# Find all tags that contain the layer(s) introducing the vulnerability,
# in repositories that have the event setup.
for layer_id in notify_layers:
# Split the layer ID into its Docker Image ID and storage ID.
(docker_image_id, storage_uuid) = layer_id.split('.', 2)
# Find the matching tags.
matching = get_matching_tags(docker_image_id, storage_uuid, RepositoryTag, Repository,
Image, ImageStorage)
tags = list(filter_tags_have_repository_event(matching, event))
check_map = {}
for tag in tags:
# Verify that the tag's root image has the vulnerability.
tag_layer_id = '%s.%s' % (tag.image.docker_image_id, tag.image.storage.uuid)
logger.debug('Checking if layer %s is vulnerable to %s', tag_layer_id, cve_id)
if not tag_layer_id in check_map:
try:
is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id)
except APIRequestFailure:
return False
check_map[tag_layer_id] = is_vulerable
logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id,
check_map[tag_layer_id])
if check_map[tag_layer_id]:
# Add the vulnerable tag to the list.
tag_map[tag.repository_id].add(tag.name)
repository_map[tag.repository_id] = tag.repository
# For each of the tags found, issue a notification.
for repository_id in tag_map:
tags = tag_map[repository_id]
event_data = {
'tags': list(tags),
'vulnerability': {
'id': cve_id,
'description': new_vuln.get('Description', None),
'link': new_vuln.get('Link', None),
'priority': new_severity['title'],
'has_fix': 'FixedIn' in new_vuln,
},
}
spawn_notification(repository_map[repository_id], 'vulnerability_found', event_data)
return True