Merge pull request #2403 from jzelinskie/clair-qss-latest

workers.securityworker: find eligible tag images

data/model/tag.py

@@ -2,7 +2,7 @@ import logging
 
 from uuid import uuid4
 
-from peewee import IntegrityError
+from peewee import IntegrityError, JOIN_LEFT_OUTER, fn
 from data.model import (image, db_transaction, DataModelException, _basequery,
                         InvalidManifestException, TagAlreadyCreatedException, StaleTagException)
 from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
@@ -13,6 +13,40 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names
 logger = logging.getLogger(__name__)
 
 
+def get_max_id_for_sec_scan():
+  """ Gets the maximum id for security scanning """
+  return RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar()
+
+
+def get_min_id_for_sec_scan(version):
+  """ Gets the minimum id for a security scanning """
+  return (RepositoryTag
+          .select(fn.Min(RepositoryTag.id))
+          .join(Image)
+          .where(Image.security_indexed_engine < version)
+          .scalar())
+
+
+def get_tag_pk_field():
+  """ Returns the primary key for Image DB model """
+  return RepositoryTag.id
+
+
+def get_tags_images_eligible_for_scan(clair_version):
+  Parent = Image.alias()
+  ParentImageStorage = ImageStorage.alias()
+
+  return _tag_alive(RepositoryTag
+                    .select(Image, ImageStorage, Parent, ParentImageStorage, RepositoryTag)
+                    .join(Image, on=(RepositoryTag.image == Image.id))
+                    .join(ImageStorage, on=(Image.storage == ImageStorage.id))
+                    .switch(Image)
+                    .join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id))
+                    .join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage))
+                    .where(RepositoryTag.hidden == False)
+                    .where(Image.security_indexed_engine < clair_version))
+
+
 def _tag_alive(query, now_ts=None):
   if now_ts is None:
     now_ts = get_epoch_timestamp()

workers/securityworker.py

@@ -6,8 +6,8 @@ import features
 from app import app, secscan_api, prometheus
 from workers.worker import Worker
 from data.database import UseThenDisconnect
-from data.model.image import (get_images_eligible_for_scan, get_max_id_for_sec_scan,
-                              get_min_id_for_sec_scan, get_image_id)
+from data.model.tag import (get_tags_images_eligible_for_scan, get_tag_pk_field,
+                            get_max_id_for_sec_scan, get_min_id_for_sec_scan)
 from util.secscan.api import SecurityConfigValidator
 from util.secscan.analyzer import LayerAnalyzer, PreemptedException
 from util.migrate.allocator import yield_random_entries
@@ -43,7 +43,7 @@ class SecurityWorker(Worker):
 
   def _index_images(self):
     def batch_query():
-      return get_images_eligible_for_scan(self._target_version)
+      return get_tags_images_eligible_for_scan(self._target_version)
 
     # Get the ID of the last image we can analyze. Will be None if there are no images in the
     # database.
@@ -56,14 +56,14 @@ class SecurityWorker(Worker):
     with UseThenDisconnect(app.config):
       to_scan_generator = yield_random_entries(
         batch_query,
-        get_image_id(),
+        get_tag_pk_field(),
         BATCH_SIZE,
         max_id,
         self._min_id,
       )
       for candidate, abt, num_remaining in to_scan_generator:
         try:
-          self._analyzer.analyze_recursively(candidate)
+          self._analyzer.analyze_recursively(candidate.image)
         except PreemptedException:
           logger.info('Another worker pre-empted us for layer: %s', candidate.id)
           abt.set()