quay/workers/securityworker.py

import logging
import logging.config

import features
import time

from peewee import fn

from app import app, config_provider, secscan_api
from workers.worker import Worker
from data.database import Image, UseThenDisconnect
from data.model.image import get_image_with_storage_and_parent_base
from util.secscan.api import SecurityConfigValidator
from util.secscan.analyzer import LayerAnalyzer
from util.migrate.allocator import yield_random_entries

BATCH_SIZE = 50
INDEXING_INTERVAL = 30

logger = logging.getLogger(__name__)

class SecurityWorker(Worker):
  def __init__(self):
    super(SecurityWorker, self).__init__()
    validator = SecurityConfigValidator(app.config, config_provider)
    if validator.valid():
      secscan_config = app.config.get('SECURITY_SCANNER')
      self._target_version = secscan_config['ENGINE_VERSION_TARGET']
      self._analyzer = LayerAnalyzer(app.config, secscan_api)

      # Get the ID of the first image we want to analyze.
      self._min_id = (Image
                      .select(fn.Min(Image.id))
                      .where(Image.security_indexed_engine < self._target_version)
                      .scalar())

      self.add_operation(self._index_images, INDEXING_INTERVAL)
    else:
      logger.warning('Failed to validate security scan configuration')

  def _index_images(self):
    def batch_query():
      base_query = get_image_with_storage_and_parent_base()
      return base_query.where(Image.security_indexed_engine < self._target_version)

    # Get the ID of the last image we can analyze.
    max_id = Image.select(fn.Max(Image.id)).scalar()

    with UseThenDisconnect(app.config):
      for candidate, abt in yield_random_entries(batch_query, Image.id, BATCH_SIZE, max_id,
                                                 self._min_id):
        _, continue_batch = self._analyzer.analyze_recursively(candidate)
        if not continue_batch:
          logger.info('Another worker pre-empted us for layer: %s', candidate.id)
          abt.set()

    # If we reach this point, we analyzed every images up to max_id, next time the worker runs,
    # we want to start from the next image.
    self._min_id = max_id + 1

if __name__ == '__main__':
  if not features.SECURITY_SCANNER:
    logger.debug('Security scanner disabled; skipping SecurityWorker')
    while True:
      time.sleep(100000)

  logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
  worker = SecurityWorker()
  worker.start()