import logging import logging.config import features import time from peewee import fn from app import app, config_provider, secscan_api from workers.worker import Worker from data.database import Image, UseThenDisconnect from data.model.image import get_image_with_storage_and_parent_base from util.secscan.api import SecurityConfigValidator from util.secscan.analyzer import LayerAnalyzer from util.migrate.allocator import yield_random_entries BATCH_SIZE = 50 INDEXING_INTERVAL = 30 logger = logging.getLogger(__name__) class SecurityWorker(Worker): def __init__(self): super(SecurityWorker, self).__init__() validator = SecurityConfigValidator(app.config, config_provider) if validator.valid(): secscan_config = app.config.get('SECURITY_SCANNER') self._target_version = secscan_config['ENGINE_VERSION_TARGET'] self._analyzer = LayerAnalyzer(app.config, secscan_api) # Get the ID of the first image we want to analyze. self._min_id = (Image .select(fn.Min(Image.id)) .where(Image.security_indexed_engine < self._target_version) .scalar()) self.add_operation(self._index_images, INDEXING_INTERVAL) else: logger.warning('Failed to validate security scan configuration') def _index_images(self): def batch_query(): base_query = get_image_with_storage_and_parent_base() return base_query.where(Image.security_indexed_engine < self._target_version) # Get the ID of the last image we can analyze. max_id = Image.select(fn.Max(Image.id)).scalar() with UseThenDisconnect(app.config): for candidate, abt in yield_random_entries(batch_query, Image.id, BATCH_SIZE, max_id, self._min_id): _, continue_batch = self._analyzer.analyze_recursively(candidate) if not continue_batch: logger.info('Another worker pre-empted us for layer: %s', candidate.id) abt.set() # If we reach this point, we analyzed every images up to max_id, next time the worker runs, # we want to start from the next image. self._min_id = max_id + 1 if __name__ == '__main__': if not features.SECURITY_SCANNER: logger.debug('Security scanner disabled; skipping SecurityWorker') while True: time.sleep(100000) logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) worker = SecurityWorker() worker.start()