Merge pull request #2372 from jakedt/clairunscanned

Move the images count stat back to the prom stat worker

data/model/image.py

@@ -1,16 +1,16 @@
 import logging
-import dateutil.parser
 import hashlib
 import json
+import dateutil.parser
 
-from peewee import JOIN_LEFT_OUTER, IntegrityError, fn
 from datetime import datetime
+from peewee import JOIN_LEFT_OUTER, IntegrityError, fn
 
 from data.model import (DataModelException, db_transaction, _basequery, storage,
-                        InvalidImageException, config)
+                        InvalidImageException)
 from data.database import (Image, Repository, ImageStoragePlacement, Namespace, ImageStorage,
                            ImageStorageLocation, RepositoryPermission, DerivedStorageForImage,
-                           ImageStorageTransformation, db_random_func)
+                           ImageStorageTransformation)
 
 from util.canonicaljson import canonicalize
 
@@ -497,15 +497,8 @@ def get_image_id():
 
 def get_images_eligible_for_scan(clair_version):
   """ Returns a query that gives all images eligible for a clair scan """
-  return get_image_with_storage_and_parent_base().where(Image.security_indexed_engine < clair_version)
+  return (get_image_with_storage_and_parent_base()
-
+          .where(Image.security_indexed_engine < clair_version))
-
-def get_count_of_images_eligible_for_scan(clair_version):
-  """ Returns a query that gives all images eligible for a clair scan """
-  # TODO(jzelinskie): Get this value from the slab allocator rather than querying the db
-  # This was the previous implementation:
-  # return get_images_eligible_for_scan(clair_version).count()
-  return 0
 
 
 def get_image_with_storage_and_parent_base():

workers/globalpromstats.py

@@ -5,9 +5,9 @@ import features
 from app import app, metric_queue
 from data.database import UseThenDisconnect
 from data import model
-from data.model.image import total_image_count, get_count_of_images_eligible_for_scan
+from data.model.image import total_image_count
 from util.locking import GlobalLock, LockNotAcquiredException
-from workers.securityworker import unscanned_images_gauge, images_gauge
+from workers.securityworker import images_gauge
 from workers.worker import Worker
 
 logger = logging.getLogger(__name__)
@@ -43,6 +43,8 @@ class GlobalPrometheusStatsWorker(Worker):
       metric_queue.org_count.Set(model.organization.get_active_org_count())
       metric_queue.robot_count.Set(model.user.get_robot_count())
 
+      images_gauge.Set(total_image_count())
+
 
 def main():
   logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)

workers/securityworker.py

@@ -7,7 +7,7 @@ from app import app, secscan_api, prometheus
 from workers.worker import Worker
 from data.database import UseThenDisconnect
 from data.model.image import (get_images_eligible_for_scan, get_max_id_for_sec_scan,
-                              get_min_id_for_sec_scan, get_image_id, total_image_count)
+                              get_min_id_for_sec_scan, get_image_id)
 from util.secscan.api import SecurityConfigValidator
 from util.secscan.analyzer import LayerAnalyzer, PreemptedException
 from util.migrate.allocator import yield_random_entries
@@ -19,7 +19,8 @@ INDEXING_INTERVAL = 30
 
 
 logger = logging.getLogger(__name__)
-unscanned_images_gauge = prometheus.create_gauge('unscanned_images', 'Number of images that clair needs to scan.')
+unscanned_images_gauge = prometheus.create_gauge('unscanned_images',
+                                                 'Number of images that clair needs to scan.')
 images_gauge = prometheus.create_gauge('all_images', 'Total number of images that clair can scan.')
 
 
@@ -64,8 +65,6 @@ class SecurityWorker(Worker):
           abt.set()
 
         unscanned_images_gauge.Set(num_remaining)
-        images_gauge.Set(total_image_count())
-
 
     # If we reach this point, we analyzed every images up to max_id, next time the worker runs,
     # we want to start from the next image.