From 88e85cded0966ad15bd8c09715157b6d5b2c4385 Mon Sep 17 00:00:00 2001 From: Quentin Machu Date: Wed, 18 Nov 2015 19:42:20 -0500 Subject: [PATCH] Fix security worker (again?) --- data/model/image.py | 40 +++++++++++++++++++-------------------- workers/securityworker.py | 9 +++------ 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/data/model/image.py b/data/model/image.py index b22f70ce3..b9954b593 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -468,13 +468,13 @@ def get_secscan_candidates(engine_version, batch_size): rimages = [] # Collect the images without parents - candidates = (Image - .select(Image.id) - .join(ImageStorage) - .where(Image.security_indexed_engine < engine_version, - Image.parent >> None, - ImageStorage.uploading == False) - .limit(batch_size*10)) + candidates = list(Image + .select(Image.id) + .join(ImageStorage) + .where(Image.security_indexed_engine < engine_version, + Image.parent >> None, + ImageStorage.uploading == False) + .limit(batch_size*10)) images = (Image .select(Image, ImageStorage) @@ -483,20 +483,19 @@ def get_secscan_candidates(engine_version, batch_size): .order_by(db_random_func()) .limit(batch_size)) - for image in images: - rimages.append(image) + rimages.extend(images) # Collect the images with analyzed parents. - candidates = (Image - .select(Image.id) - .join(Parent, on=(Image.parent == Parent.id)) - .switch(Image) - .join(ImageStorage) - .where(Image.security_indexed_engine < engine_version, - Parent.security_indexed == True, - Parent.security_indexed_engine >= engine_version, - ImageStorage.uploading == False) - .limit(batch_size*10)) + candidates = list(Image + .select(Image.id) + .join(Parent, on=(Image.parent == Parent.id)) + .switch(Image) + .join(ImageStorage) + .where(Image.security_indexed_engine < engine_version, + Parent.security_indexed == True, + Parent.security_indexed_engine >= engine_version, + ImageStorage.uploading == False) + .limit(batch_size*10)) images = (Image .select(Image, ImageStorage, Parent, ParentImageStorage) @@ -508,8 +507,7 @@ def get_secscan_candidates(engine_version, batch_size): .order_by(db_random_func()) .limit(batch_size)) - for image in images: - rimages.append(image) + rimages.extend(images) # Shuffle the images, otherwise the images without parents will always be on the top random.shuffle(rimages) diff --git a/workers/securityworker.py b/workers/securityworker.py index 8db7bf157..968de7cbd 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -13,7 +13,7 @@ from data import model from data.model.tag import filter_tags_have_repository_event, get_tags_for_image from data.model.image import get_secscan_candidates, set_secscan_status from data.model.storage import get_storage_locations -from data.database import (Image, UseThenDisconnect, ExternalNotificationEvent) +from data.database import (UseThenDisconnect, ExternalNotificationEvent) from util.secscan.api import SecurityConfigValidator logger = logging.getLogger(__name__) @@ -154,11 +154,8 @@ class SecurityWorker(Worker): while True: # Lookup the images to index. images = [] - try: - logger.debug('Looking up images to index') - images = get_secscan_candidates(self._target_version, BATCH_SIZE) - except Image.DoesNotExist: - pass + logger.debug('Looking up images to index') + images = get_secscan_candidates(self._target_version, BATCH_SIZE) if not images: logger.debug('No more images left to analyze')