Unify the database connection lifecycle across all workers
This commit is contained in:
Jake Moshenko
parent
38cb63d195
commit
2f626f2691
7 changed files with 111 additions and 130 deletions
|
|
@ -13,7 +13,7 @@ from data import model
|
|||
from data.model.tag import filter_tags_have_repository_event, get_tags_for_image
|
||||
from data.model.image import get_secscan_candidates, set_secscan_status
|
||||
from data.model.storage import get_storage_locations
|
||||
from data.database import (UseThenDisconnect, ExternalNotificationEvent)
|
||||
from data.database import ExternalNotificationEvent
|
||||
from util.secscan.api import SecurityConfigValidator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -150,68 +150,67 @@ class SecurityWorker(Worker):
|
|||
logger.debug('Started indexing')
|
||||
event = ExternalNotificationEvent.get(name='vulnerability_found')
|
||||
|
||||
with UseThenDisconnect(app.config):
|
||||
while True:
|
||||
# Lookup the images to index.
|
||||
images = []
|
||||
logger.debug('Looking up images to index')
|
||||
images = get_secscan_candidates(self._target_version, BATCH_SIZE)
|
||||
while True:
|
||||
# Lookup the images to index.
|
||||
images = []
|
||||
logger.debug('Looking up images to index')
|
||||
images = get_secscan_candidates(self._target_version, BATCH_SIZE)
|
||||
|
||||
if not images:
|
||||
logger.debug('No more images left to analyze')
|
||||
if not images:
|
||||
logger.debug('No more images left to analyze')
|
||||
return
|
||||
|
||||
logger.debug('Found %d images to index', len(images))
|
||||
for image in images:
|
||||
# If we couldn't analyze the parent, we can't analyze this image.
|
||||
if (image.parent and not image.parent.security_indexed and
|
||||
image.parent.security_indexed_engine >= self._target_version):
|
||||
set_secscan_status(image, False, self._target_version)
|
||||
continue
|
||||
|
||||
# Analyze the image.
|
||||
analyzed = self._analyze_image(image)
|
||||
if not analyzed:
|
||||
return
|
||||
|
||||
logger.debug('Found %d images to index', len(images))
|
||||
for image in images:
|
||||
# If we couldn't analyze the parent, we can't analyze this image.
|
||||
if (image.parent and not image.parent.security_indexed and
|
||||
image.parent.security_indexed_engine >= self._target_version):
|
||||
set_secscan_status(image, False, self._target_version)
|
||||
# Get the tags of the image we analyzed
|
||||
matching = list(filter_tags_have_repository_event(get_tags_for_image(image.id), event))
|
||||
|
||||
repository_map = defaultdict(list)
|
||||
|
||||
for tag in matching:
|
||||
repository_map[tag.repository_id].append(tag)
|
||||
|
||||
# If there is at least one tag,
|
||||
# Lookup the vulnerabilities for the image, now that it is analyzed.
|
||||
if len(repository_map) > 0:
|
||||
logger.debug('Loading vulnerabilities for layer %s', image.id)
|
||||
sec_data = self._get_vulnerabilities(image)
|
||||
|
||||
if sec_data is None:
|
||||
continue
|
||||
|
||||
# Analyze the image.
|
||||
analyzed = self._analyze_image(image)
|
||||
if not analyzed:
|
||||
return
|
||||
if not sec_data.get('Vulnerabilities'):
|
||||
continue
|
||||
|
||||
# Get the tags of the image we analyzed
|
||||
matching = list(filter_tags_have_repository_event(get_tags_for_image(image.id), event))
|
||||
# Dispatch events for any detected vulnerabilities
|
||||
logger.debug('Got vulnerabilities for layer %s: %s', image.id, sec_data)
|
||||
|
||||
repository_map = defaultdict(list)
|
||||
for repository_id in repository_map:
|
||||
tags = repository_map[repository_id]
|
||||
|
||||
for tag in matching:
|
||||
repository_map[tag.repository_id].append(tag)
|
||||
for vuln in sec_data['Vulnerabilities']:
|
||||
event_data = {
|
||||
'tags': [tag.name for tag in tags],
|
||||
'vulnerability': {
|
||||
'id': vuln['ID'],
|
||||
'description': vuln['Description'],
|
||||
'link': vuln['Link'],
|
||||
'priority': vuln['Priority'],
|
||||
},
|
||||
}
|
||||
|
||||
# If there is at least one tag,
|
||||
# Lookup the vulnerabilities for the image, now that it is analyzed.
|
||||
if len(repository_map) > 0:
|
||||
logger.debug('Loading vulnerabilities for layer %s', image.id)
|
||||
sec_data = self._get_vulnerabilities(image)
|
||||
|
||||
if sec_data is None:
|
||||
continue
|
||||
|
||||
if not sec_data.get('Vulnerabilities'):
|
||||
continue
|
||||
|
||||
# Dispatch events for any detected vulnerabilities
|
||||
logger.debug('Got vulnerabilities for layer %s: %s', image.id, sec_data)
|
||||
|
||||
for repository_id in repository_map:
|
||||
tags = repository_map[repository_id]
|
||||
|
||||
for vuln in sec_data['Vulnerabilities']:
|
||||
event_data = {
|
||||
'tags': [tag.name for tag in tags],
|
||||
'vulnerability': {
|
||||
'id': vuln['ID'],
|
||||
'description': vuln['Description'],
|
||||
'link': vuln['Link'],
|
||||
'priority': vuln['Priority'],
|
||||
},
|
||||
}
|
||||
|
||||
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
|
||||
spawn_notification(tags[0].repository, 'vulnerability_found', event_data)
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not features.SECURITY_SCANNER:
|
||||
|
|
|
|||
Reference in a new issue