Security scanner garbage collection support
Adds support for calling GC in the security scanner for any layers+storage removed by GC on the Quay side
This commit is contained in:
Joseph Schorr
parent
5225642850
commit
d609e6a1c4
4 changed files with 87 additions and 23 deletions
|
|
@ -1,8 +1,10 @@
|
|||
import logging
|
||||
|
||||
from urlparse import urljoin
|
||||
|
||||
import requests
|
||||
|
||||
from flask import url_for
|
||||
from urlparse import urljoin
|
||||
|
||||
from data.database import CloseForLongOperation
|
||||
from data import model
|
||||
|
|
@ -40,11 +42,17 @@ class APIRequestFailure(Exception):
|
|||
|
||||
_API_METHOD_INSERT = 'layers'
|
||||
_API_METHOD_GET_LAYER = 'layers/%s'
|
||||
_API_METHOD_DELETE_LAYER = 'layers/%s'
|
||||
_API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s'
|
||||
_API_METHOD_GET_NOTIFICATION = 'notifications/%s'
|
||||
_API_METHOD_PING = 'metrics'
|
||||
|
||||
|
||||
def compute_layer_id(layer):
|
||||
""" Returns the ID for the layer in the security scanner. """
|
||||
return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
|
||||
|
||||
|
||||
class SecurityScannerAPI(object):
|
||||
""" Helper class for talking to the Security Scan service (Clair). """
|
||||
def __init__(self, app, config, storage, client=None, skip_validation=False):
|
||||
|
|
@ -62,7 +70,6 @@ class SecurityScannerAPI(object):
|
|||
self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
|
||||
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
|
||||
|
||||
|
||||
def _get_image_url_and_auth(self, image):
|
||||
""" Returns a tuple of the url and the auth header value that must be used
|
||||
to fetch the layer data itself. If the image can't be addressed, we return
|
||||
|
|
@ -74,8 +81,8 @@ class SecurityScannerAPI(object):
|
|||
if not self._storage.exists(locations, path):
|
||||
locations = get_storage_locations(image.storage.uuid)
|
||||
if not locations or not self._storage.exists(locations, path):
|
||||
logger.warning('Could not find a valid location to download layer %s.%s out of %s',
|
||||
image.docker_image_id, image.storage.uuid, locations)
|
||||
logger.warning('Could not find a valid location to download layer %s out of %s',
|
||||
compute_layer_id(image), locations)
|
||||
return None, None
|
||||
|
||||
uri = self._storage.get_direct_download_url(locations, path)
|
||||
|
|
@ -106,17 +113,16 @@ class SecurityScannerAPI(object):
|
|||
|
||||
return uri, auth_header
|
||||
|
||||
|
||||
def _new_analyze_request(self, image):
|
||||
""" Create the request body to submit the given image for analysis. If the image's URL cannot
|
||||
def _new_analyze_request(self, layer):
|
||||
""" Create the request body to submit the given layer for analysis. If the layer's URL cannot
|
||||
be found, returns None.
|
||||
"""
|
||||
url, auth_header = self._get_image_url_and_auth(image)
|
||||
url, auth_header = self._get_image_url_and_auth(layer)
|
||||
if url is None:
|
||||
return None
|
||||
|
||||
layer_request = {
|
||||
'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid),
|
||||
'Name': compute_layer_id(layer),
|
||||
'Path': url,
|
||||
'Format': 'Docker',
|
||||
}
|
||||
|
|
@ -126,14 +132,23 @@ class SecurityScannerAPI(object):
|
|||
'Authorization': auth_header,
|
||||
}
|
||||
|
||||
if image.parent.docker_image_id and image.parent.storage.uuid:
|
||||
layer_request['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
|
||||
image.parent.storage.uuid)
|
||||
if layer.parent.docker_image_id and layer.parent.storage.uuid:
|
||||
layer_request['ParentName'] = compute_layer_id(layer.parent)
|
||||
|
||||
return {
|
||||
'Layer': layer_request,
|
||||
}
|
||||
|
||||
def cleanup_layers(self, layers):
|
||||
""" Callback invoked by garbage collection to cleanup any layers that no longer
|
||||
need to be stored in the security scanner.
|
||||
"""
|
||||
if self._config is None:
|
||||
# Security scanner not enabled.
|
||||
return
|
||||
|
||||
for layer in layers:
|
||||
self.delete_layer(layer)
|
||||
|
||||
def ping(self):
|
||||
""" Calls GET on the metrics endpoint of the security scanner to ensure it is running
|
||||
|
|
@ -151,6 +166,17 @@ class SecurityScannerAPI(object):
|
|||
logger.exception('Exception when trying to connect to security scanner endpoint')
|
||||
raise Exception('Exception when trying to connect to security scanner endpoint')
|
||||
|
||||
def delete_layer(self, layer):
|
||||
""" Calls DELETE on the given layer in the security scanner, removing it from
|
||||
its database.
|
||||
"""
|
||||
layer_id = compute_layer_id(layer)
|
||||
try:
|
||||
response = self._call('DELETE', _API_METHOD_DELETE_LAYER % layer_id)
|
||||
return response.status_code / 100 == 2
|
||||
except requests.exceptions.RequestException:
|
||||
logger.exception('Failed to delete layer: %s', layer_id)
|
||||
return False
|
||||
|
||||
def analyze_layer(self, layer):
|
||||
""" Posts the given layer to the security scanner for analysis, blocking until complete.
|
||||
|
|
@ -201,7 +227,6 @@ class SecurityScannerAPI(object):
|
|||
# Return the parsed API version.
|
||||
return json_response['Layer']['IndexedByVersion']
|
||||
|
||||
|
||||
def check_layer_vulnerable(self, layer_id, cve_name):
|
||||
""" Checks to see if the layer with the given ID is vulnerable to the specified CVE. """
|
||||
layer_data = self._get_layer_data(layer_id, include_vulnerabilities=True)
|
||||
|
|
@ -215,7 +240,6 @@ class SecurityScannerAPI(object):
|
|||
|
||||
return False
|
||||
|
||||
|
||||
def get_notification(self, notification_name, layer_limit=100, page=None):
|
||||
""" Gets the data for a specific notification, with optional page token.
|
||||
Returns a tuple of the data (None on failure) and whether to retry.
|
||||
|
|
@ -245,7 +269,6 @@ class SecurityScannerAPI(object):
|
|||
|
||||
return json_response, False
|
||||
|
||||
|
||||
def mark_notification_read(self, notification_name):
|
||||
""" Marks a security scanner notification as read. """
|
||||
try:
|
||||
|
|
@ -255,13 +278,11 @@ class SecurityScannerAPI(object):
|
|||
logger.exception('Failed to mark notification as read: %s', notification_name)
|
||||
return False
|
||||
|
||||
|
||||
def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
|
||||
""" Returns the layer data for the specified layer. On error, returns None. """
|
||||
layer_id = '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
|
||||
layer_id = compute_layer_id(layer)
|
||||
return self._get_layer_data(layer_id, include_features, include_vulnerabilities)
|
||||
|
||||
|
||||
def _get_layer_data(self, layer_id, include_features=False, include_vulnerabilities=False):
|
||||
try:
|
||||
params = {}
|
||||
|
|
@ -288,7 +309,6 @@ class SecurityScannerAPI(object):
|
|||
|
||||
return json_response
|
||||
|
||||
|
||||
def _call(self, method, relative_url, params=None, body=None):
|
||||
""" Issues an HTTP call to the sec API at the given relative URL.
|
||||
This function disconnects from the database while awaiting a response
|
||||
|
|
|
|||
Reference in a new issue