Merge pull request #894 from Quentin-M/fix_secwor

Refactor security worker

data/model/image.py

@@ -1,5 +1,6 @@
 import logging
 import dateutil.parser
+import random
 
 from peewee import JOIN_LEFT_OUTER, fn, SQL
 from datetime import datetime
@@ -7,7 +8,8 @@ from datetime import datetime
 from data.model import (DataModelException, db_transaction, _basequery, storage,
                         InvalidImageException, config)
 from data.database import (Image, Repository, ImageStoragePlacement, Namespace, ImageStorage,
-                           ImageStorageLocation, RepositoryPermission, db_for_update)
+                           ImageStorageLocation, RepositoryPermission, db_for_update,
+                           db_random_func)
 
 
 logger = logging.getLogger(__name__)
@@ -459,3 +461,74 @@ def ensure_image_locations(*names):
 
     data = [{'name': name} for name in insert_names]
     ImageStorageLocation.insert_many(data).execute()
+
+def get_secscan_candidates(engine_version, batch_size):
+  Parent = Image.alias()
+  ParentImageStorage = ImageStorage.alias()
+  rimages = []
+
+  # Collect the images without parents
+  candidates = (Image
+                .select(Image.id)
+                .join(ImageStorage)
+                .where(Image.security_indexed_engine < engine_version,
+                       Image.parent >> None,
+                       ImageStorage.uploading == False)
+                .limit(batch_size*10))
+
+  images = (Image
+            .select(Image, ImageStorage)
+            .join(ImageStorage)
+            .where(Image.id << candidates)
+            .order_by(db_random_func())
+            .limit(batch_size))
+
+  for image in images:
+    rimages.append(image)
+
+  # Collect the images with analyzed parents.
+  candidates = (Image
+                .select(Image.id)
+                .join(Parent, on=(Image.parent == Parent.id))
+                .switch(Image)
+                .join(ImageStorage)
+                .where(Image.security_indexed_engine < engine_version,
+                       Parent.security_indexed == True,
+                       Parent.security_indexed_engine >= engine_version,
+                       ImageStorage.uploading == False)
+                .limit(batch_size*10))
+
+  images = (Image
+            .select(Image, ImageStorage, Parent, ParentImageStorage)
+            .join(Parent, on=(Image.parent == Parent.id))
+            .join(ParentImageStorage, on=(ParentImageStorage.id == Parent.storage))
+            .switch(Image)
+            .join(ImageStorage)
+            .where(Image.id << candidates)
+            .order_by(db_random_func())
+            .limit(batch_size))
+
+  for image in images:
+    rimages.append(image)
+
+  # Shuffle the images, otherwise the images without parents will always be on the top
+  random.shuffle(rimages)
+
+  return rimages
+
+
+def set_secscan_status(image, indexed, version):
+  query = (Image
+           .select()
+           .join(ImageStorage)
+           .where(Image.docker_image_id == image.docker_image_id,
+                  ImageStorage.uuid == image.storage.uuid))
+
+  ids_to_update = [row.id for row in query]
+  if not ids_to_update:
+    return
+
+  (Image
+   .update(security_indexed=indexed, security_indexed_engine=version)
+   .where(Image.id << ids_to_update)
+   .execute())

data/model/storage.py

@@ -225,3 +225,13 @@ def lookup_repo_storages_by_content_checksum(repo, checksums):
           .select()
           .join(Image)
           .where(Image.repository == repo, ImageStorage.content_checksum << checksums))
+
+def get_storage_locations(uuid):
+  query = (ImageStoragePlacement
+           .select()
+           .join(ImageStorageLocation)
+           .switch(ImageStoragePlacement)
+           .join(ImageStorage, JOIN_LEFT_OUTER)
+           .where(ImageStorage.uuid == uuid))
+
+  return [location.location.name for location in query]

data/model/tag.py

@@ -3,7 +3,7 @@ from uuid import uuid4
 from data.model import (image, db_transaction, DataModelException, _basequery,
                         InvalidManifestException)
 from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
-                           get_epoch_timestamp, db_for_update)
+                           RepositoryNotification, get_epoch_timestamp, db_for_update)
 
 
 def _tag_alive(query, now_ts=None):
@@ -18,15 +18,29 @@ def get_matching_tags(docker_image_id, storage_uuid, *args):
       given docker_image_id and storage_uuid. """
   image_query = image.get_repository_image_and_deriving(docker_image_id, storage_uuid)
 
-  return (RepositoryTag
-            .select(*args)
-            .distinct()
-            .join(Image)
-            .join(ImageStorage)
-            .where(Image.id << image_query, RepositoryTag.lifetime_end_ts >> None,
-                   RepositoryTag.hidden == False))
+  return _tag_alive(RepositoryTag
+                   .select(*args)
+                   .distinct()
+                   .join(Image)
+                   .join(ImageStorage)
+                   .where(Image.id << image_query, RepositoryTag.hidden == False))
 
 
+def get_tags_for_image(image_id, *args):
+  return _tag_alive(RepositoryTag
+                    .select(*args)
+                    .distinct()
+                    .where(RepositoryTag.image == image_id,
+                           RepositoryTag.hidden == False))
+
+
+def filter_tags_have_repository_event(query, event):
+  return (query
+          .switch(RepositoryTag)
+          .join(Repository)
+          .join(RepositoryNotification)
+          .where(RepositoryNotification.event == event))
+
 def list_repository_tags(namespace_name, repository_name, include_hidden=False,
                          include_storage=False):
   to_select = (RepositoryTag, Image)
@@ -233,4 +247,3 @@ def _load_repo_manifests(namespace, repo_name):
             .join(Repository)
             .join(Namespace, on=(Namespace.id == Repository.namespace_user))
             .where(Repository.name == repo_name, Namespace.username == namespace))
-