vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix all the things!

Browse source
This commit is contained in:
Joseph Schorr 2015-11-12 17:47:19 -05:00
parent 399bf4e9c7
commit 25b8b7590f
4 changed files with 58 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
data/model/tag.py
View file

@ -22,7 +22,8 @@ def get_matching_tags(docker_image_id, storage_uuid, *args):
.distinct()
.join(Image)
.join(ImageStorage)
.where(Image.id << image_query, RepositoryTag.lifetime_end_ts >> None))
.where(Image.id << image_query, RepositoryTag.lifetime_end_ts >> None,
RepositoryTag.hidden == False))
def list_repository_tags(namespace_name, repository_name, include_hidden=False,

28
util/secscan/api.py
View file

@ -134,12 +134,24 @@ class SecurityConfigValidator(object):
return self._keys
def valid(self):
if (not features.SECURITY_SCANNER
or not self._security_config
or not 'ENDPOINT' in self._security_config
or not 'ENGINE_VERSION_TARGET' in self._security_config
or not 'DISTRIBUTED_STORAGE_PREFERENCE' in self._security_config
or (self._certificate is False and self._keys is None)):
if not features.SECURITY_SCANNER:
return False
if not self._security_config:
logger.debug('Missing SECURITY_SCANNER block in configuration')
return False
if not 'ENDPOINT' in self._security_config:
logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration')
return False
endpoint = self._security_config['ENDPOINT'] or ''
if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https')
return False
if endpoint.startswith('https://') and (self._certificate is False or self._keys is None):
logger.debug('Certificate and key pair required for talking to security worker over HTTPS')
return False
return True
@ -150,6 +162,7 @@ class SecurityScannerAPI(object):
def __init__(self, app, config_provider):
self.app = app
self.config_provider = config_provider
self._security_config = None
config_validator = SecurityConfigValidator(app, config_provider)
if not config_validator.valid():
@ -192,6 +205,9 @@ class SecurityScannerAPI(object):
from the API server.
"""
security_config = self._security_config
if security_config is None:
raise Exception('Cannot call unconfigured security system')
api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
url = urljoin(api_url, relative_url % args)

4
workers/security_notification_worker.py
View file

@ -18,9 +18,7 @@ logger = logging.getLogger(__name__)
class SecurityNotificationWorker(QueueWorker):
def process_queue_item(self, queueitem):
data = json.loads(queueitem.body)
def process_queue_item(self, data):
cve_id = data['Name']
vulnerability = data['Content']['Vulnerability']
priority = vulnerability['Priority']

68
workers/securityworker.py
View file

@ -119,7 +119,7 @@ def _update_image(image, indexed, version):
class SecurityWorker(Worker):
def __init__(self):
super(SecurityWorker, self).__init__()
validator = SecurityConfigValidator(app.config, config_provider)
validator = SecurityConfigValidator(app, config_provider)
if validator.valid():
secscan_config = app.config.get('SECURITY_SCANNER')
self._api = secscan_config['ENDPOINT']
@ -143,10 +143,7 @@ class SecurityWorker(Worker):
if not storage.exists(locations, path):
logger.warning('Could not find a valid location to download layer %s',
image['docker_image_id']+'.'+image['storage_uuid'])
try:
_update_image(image, False, self._target_version)
except:
logger.exception('Failed to update unindexed image')
_update_image(image, False, self._target_version)
return None
uri = storage.get_direct_download_url(locations, path)
@ -182,10 +179,14 @@ class SecurityWorker(Worker):
return request
def _analyze_image(self, image):
""" Analyzes an image by passing it to Clair. Returns the vulnerabilities detected
(if any) or None on error.
"""
request = self._new_request(image)
if request is None:
return None
# Analyze the image.
try:
logger.info('Analyzing %s', request['ID'])
# Using invalid certificates doesn't return proper errors because of
@ -199,43 +200,36 @@ class SecurityWorker(Worker):
# Handle any errors from the security scanner.
if httpResponse.status_code != 201:
if 'Message' in jsonResponse:
if 'OS and/or package manager are not supported' in jsonResponse['Message']:
# The current engine could not index this layer
logger.warning('A warning event occurred when analyzing layer ID %s : %s',
request['ID'], jsonResponse['Message'])
if 'OS and/or package manager are not supported' in jsonResponse.get('Message', ''):
# The current engine could not index this layer
logger.warning('A warning event occurred when analyzing layer ID %s : %s',
request['ID'], jsonResponse['Message'])
# Hopefully, there is no version lower than the target one running
try:
_update_image(image, False, self._target_version)
except:
logger.exception('Failed to update image to be unindexed')
else:
logger.warning('Failed to handle JSON message "%s" when analyzing layer ID %s',
jsonResponse['Message'], request['ID'])
return None
# Hopefully, there is no version lower than the target one running
_update_image(image, False, self._target_version)
else:
logger.warning('No message found in JSON response when analyzing layer ID %s', request['ID'])
return None
logger.warning('Got non-201 when analyzing layer ID %s: %s', request['ID'], jsonResponse)
api_version = jsonResponse['Version']
if api_version < self._target_version:
logger.warning('An engine runs on version %d but the target version is %d')
return None
try:
_update_image(image, True, api_version)
logger.debug('Layer %s analyzed successfully', request['ID'])
except:
logger.exception('Failed to update image to be indexed')
# Verify that the version matches.
api_version = jsonResponse['Version']
if api_version < self._target_version:
logger.warning('An engine runs on version %d but the target version is %d')
logger.debug('Loading vulnerabilities for layer %s', image['image_id'])
# Mark the image as analyzed.
logger.debug('Layer %s analyzed successfully; Loading vulnerabilities for layer',
image['image_id'])
_update_image(image, True, api_version)
# Lookup the vulnerabilities for the image, now that it is analyzed.
try:
response = secscan_api.call('layers/%s/vulnerabilities', None, request['ID'])
logger.debug('Got response %s for vulnerabilities for layer %s',
response.status_code, image['image_id'])
if response.status_code == 404:
return None
except:
except (requests.exceptions.RequestException, ValueError):
logger.exception('Failed to get vulnerability response for %s', image['image_id'])
return None
@ -246,6 +240,7 @@ class SecurityWorker(Worker):
with UseThenDisconnect(app.config):
while True:
# Lookup the images to index.
images = []
try:
logger.debug('Looking up images to index')
@ -256,9 +251,10 @@ class SecurityWorker(Worker):
if not images:
logger.debug('No more images left to analyze')
return
logger.debug('Found %d images to index', len(images))
logger.debug('Found %d images to index', len(images))
for image in images:
# Analyze the image, retrieving the vulnerabilities (if any).
sec_data = self._analyze_image(image)
if sec_data is None:
continue
@ -267,7 +263,7 @@ class SecurityWorker(Worker):
continue
# Dispatch events for any detected vulnerabilities
logger.debug('Got response vulnerabilities for layer %s: %s', image['image_id'], sec_data)
logger.debug('Got vulnerabilities for layer %s: %s', image['image_id'], sec_data)
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = (RepositoryTag
.select(RepositoryTag, Repository)
@ -275,9 +271,11 @@ class SecurityWorker(Worker):
.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event,
RepositoryTag.image == image['image_id']))
RepositoryTag.image == image['image_id'],
RepositoryTag.hidden == False,
RepositoryTag.lifetime_end_ts >> None))
repository_map = defaultdict()
repository_map = defaultdict(list)
for tag in matching:
repository_map[tag.repository_id].append(tag)