vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Ready for demo

Browse source
This commit is contained in:
Joseph Schorr 2015-10-28 16:32:46 -04:00 • committed by Jimmy Zelinskie
parent 75173d5573
commit b408cfd2cc
7 changed files with 94 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

139
workers/securityworker.py
View file

@ -97,13 +97,13 @@ def _update_image(image, indexed, version):
.where(Image.docker_image_id == image['docker_image_id'], ImageStorage.uuid == image['storage_uuid']))
updated_images = list()
for image in query:
updated_images.append(image.id)
for row in query:
updated_images.append(row.id)
query = (Image
(Image
.update(security_indexed=indexed, security_indexed_engine=version)
.where(Image.id << updated_images))
query.execute()
.where(Image.id << updated_images)
.execute())
class SecurityWorker(Worker):
def __init__(self):
@ -163,14 +163,17 @@ class SecurityWorker(Worker):
# Get layer storage URL
path = storage.image_layer_path(img['storage_uuid'])
locations = self._default_storage_locations
if not storage.exists(locations, path):
locations = _get_storage_locations(img['storage_uuid'])
if not storage.exists(locations, path):
logger.warning('Could not find a valid location to download layer %s', img['docker_image_id']+'.'+img['storage_uuid'])
# Mark as analyzed because that error is most likely to occur during the pre-process, with the database copy
# when images are actually removed on the real database (and therefore in S3)
_update_image(img, False, self._target_version)
continue
uri = storage.get_direct_download_url(locations, path)
if uri == None:
# Local storage hack
@ -200,69 +203,7 @@ class SecurityWorker(Worker):
logger.exception('An exception occurred when analyzing layer ID %s : the response is not valid JSON (%s)', request['ID'], httpResponse.text)
return
if httpResponse.status_code == 201:
# The layer has been successfully indexed
api_version = jsonResponse['Version']
if api_version < self._target_version:
logger.warning('An engine runs on version %d but the target version is %d')
_update_image(img, True, api_version)
logger.info('Layer ID %s : analyzed successfully', request['ID'])
# TODO(jschorr): Put this in a proper place, properly comment, unify with the
# callback code, etc.
try:
logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
except requests.exceptions.Timeout:
logger.debug('Timeout when calling Sec')
continue
except requests.exceptions.ConnectionError:
logger.debug('Connection error when calling Sec')
continue
logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
if response.status_code == 404:
continue
sec_data = json.loads(response.text)
logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
if not sec_data['Vulnerabilities']:
continue
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = (RepositoryTag
.select(RepositoryTag, Repository)
.distinct()
.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event,
RepositoryTag.image == img['image_id']))
repository_map = defaultdict(list)
for tag in matching:
repository_map[tag.repository_id].append(tag)
for repository_id in repository_map:
tags = repository_map[repository_id]
for vuln in sec_data['Vulnerabilities']:
event_data = {
'tags': [tag.name for tag in tags],
'vulnerability': {
'id': vuln['ID'],
'description': vuln['Description'],
'link': vuln['Link'],
'priority': vuln['Priority'],
},
}
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
else:
if httpResponse.status_code != 201:
if 'Message' in jsonResponse:
if 'OS and/or package manager are not supported' in jsonResponse['Message']:
# The current engine could not index this layer
@ -276,6 +217,68 @@ class SecurityWorker(Worker):
logger.exception('An exception occurred when analyzing layer ID %s : %d', request['ID'], httpResponse.status_code)
return
# The layer has been successfully indexed
api_version = jsonResponse['Version']
if api_version < self._target_version:
logger.warning('An engine runs on version %d but the target version is %d')
logger.debug('Layer %s analyzed successfully', request['ID'])
_update_image(img, True, api_version)
# TODO(jschorr): Put this in a proper place, properly comment, unify with the
# callback code, etc.
try:
logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
response = sec_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
except requests.exceptions.Timeout:
logger.debug('Timeout when calling Sec')
continue
except requests.exceptions.ConnectionError:
logger.debug('Connection error when calling Sec')
continue
logger.debug('Got response %s for vulnerabilities for layer %s', response.status_code, img['image_id'])
if response.status_code == 404:
continue
sec_data = json.loads(response.text)
logger.debug('Got response vulnerabilities for layer %s: %s', img['image_id'], sec_data)
if not sec_data['Vulnerabilities']:
continue
event = ExternalNotificationEvent.get(name='vulnerability_found')
matching = (RepositoryTag
.select(RepositoryTag, Repository)
.distinct()
.join(Repository)
.join(RepositoryNotification)
.where(RepositoryNotification.event == event,
RepositoryTag.image == img['image_id']))
repository_map = defaultdict(list)
for tag in matching:
repository_map[tag.repository_id].append(tag)
for repository_id in repository_map:
tags = repository_map[repository_id]
for vuln in sec_data['Vulnerabilities']:
event_data = {
'tags': [tag.name for tag in tags],
'vulnerability': {
'id': vuln['ID'],
'description': vuln['Description'],
'link': vuln['Link'],
'priority': vuln['Priority'],
},
}
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
if __name__ == '__main__':
if not features.SECURITY_SCANNER:
logger.debug('Security scanner disabled; skipping')