vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Change the security notification system to use get_matching_tags_for_images

Browse source 
This should vastly reduce the number of database calls we make, as instead of making 2-3 calls per image, we'll make two calls per ~100 images
This commit is contained in:
Joseph Schorr 2017-05-02 15:39:27 -04:00
parent 74dd0ef8e8
commit 98fcae753b
1 changed files with 22 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
util/secscan/notifier.py
View file

@ -5,7 +5,7 @@ from collections import defaultdict
from enum import Enum from enum import Enum
from app import secscan_api from app import secscan_api
from data.model.tag import filter_tags_have_repository_event, get_matching_tags from data.model.tag import filter_tags_have_repository_event, get_matching_tags_for_images
from data.database import (Image, ImageStorage, ExternalNotificationEvent, Repository, from data.database import (Image, ImageStorage, ExternalNotificationEvent, Repository,
RepositoryTag) RepositoryTag)
from endpoints.notificationhelper import notification_batch from endpoints.notificationhelper import notification_batch
@ -33,10 +33,10 @@ class SecurityNotificationHandler(object):
self.tag_map = defaultdict(set) self.tag_map = defaultdict(set)
self.repository_map = {} self.repository_map = {}
self.check_map = {} self.check_map = {}
self.layer_ids = set()
self.stream_tracker = None self.stream_tracker = None
self.results_per_stream = results_per_stream self.results_per_stream = results_per_stream
self.reporting_failed = False
self.vulnerability_info = None self.vulnerability_info = None
self.event = ExternalNotificationEvent.get(name='vulnerability_found') self.event = ExternalNotificationEvent.get(name='vulnerability_found')
@ -133,10 +133,6 @@ class SecurityNotificationHandler(object):
self.stream_tracker.push_new(new_layer_ids) self.stream_tracker.push_new(new_layer_ids)
self.stream_tracker.push_old(old_layer_ids) self.stream_tracker.push_old(old_layer_ids)
# If the reporting failed at any point, nothing more we can do.
if self.reporting_failed:
return ProcessNotificationPageResult.FAILED
# Check to see if there are any additional pages to process. # Check to see if there are any additional pages to process.
if 'NextPage' not in notification_page_data: if 'NextPage' not in notification_page_data:
return self._done() return self._done()
@ -145,21 +141,30 @@ class SecurityNotificationHandler(object):
def _done(self): def _done(self):
if self.stream_tracker is not None: if self.stream_tracker is not None:
# Mark the tracker as done, so that it finishes reporting any outstanding layers.
self.stream_tracker.done() self.stream_tracker.done()
if self.reporting_failed: # Process all the layers.
if self.vulnerability_info is not None:
if not self._process_layers():
return ProcessNotificationPageResult.FAILED return ProcessNotificationPageResult.FAILED
return ProcessNotificationPageResult.FINISHED_PROCESSING return ProcessNotificationPageResult.FINISHED_PROCESSING
def _report(self, new_layer_id): def _report(self, new_layer_id):
# Split the layer ID into its Docker Image ID and storage ID. self.layer_ids.add(new_layer_id)
(docker_image_id, storage_uuid) = new_layer_id.split('.', 2)
def _process_layers(self):
# Builds the pairs of layer ID and storage uuid.
pairs = [tuple(layer_id.split('.', 2)) for layer_id in self.layer_ids]
def filter_notifying_repos(query):
query = query.join(ImageStorage)
return filter_tags_have_repository_event(query, self.event)
# Find the matching tags. # Find the matching tags.
matching = get_matching_tags(docker_image_id, storage_uuid, RepositoryTag, Repository, tags = get_matching_tags_for_images(pairs, selections=[RepositoryTag, Image, ImageStorage],
Image, ImageStorage) filter_query=filter_notifying_repos)
tags = list(filter_tags_have_repository_event(matching, self.event))
cve_id = self.vulnerability_info['Name'] cve_id = self.vulnerability_info['Name']
for tag in tags: for tag in tags:
@ -170,12 +175,14 @@ class SecurityNotificationHandler(object):
try: try:
self.check_map[tag_layer_id] = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id) self.check_map[tag_layer_id] = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id)
except APIRequestFailure: except APIRequestFailure:
self.reporting_failed = True return False
return
logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id, logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id,
self.check_map[tag_layer_id]) self.check_map[tag_layer_id])
if self.check_map[tag_layer_id]: if self.check_map[tag_layer_id]:
# Add the vulnerable tag to the list. # Add the vulnerable tag to the list.
self.tag_map[tag.repository_id].add(tag.name) self.tag_map[tag.repository_id].add(tag.name)
self.repository_map[tag.repository_id] = tag.repository self.repository_map[tag.repository_id] = tag.repository
return True