Merge pull request #2183 from charltonaustin/metrics_for_unscanned_images

Adding in some metrics around clair sec scan.

workers/globalpromstats.py

@@ -1,10 +1,13 @@
 import logging
 import time
+import features
 
 from app import app, metric_queue
 from data.database import UseThenDisconnect
 from data import model
+from data.model.image import total_image_count, get_count_of_images_eligible_for_scan
 from util.locking import GlobalLock, LockNotAcquiredException
+from workers.securityworker import unscanned_images_gauge, images_gauge
 from workers.worker import Worker
 
 logger = logging.getLogger(__name__)
@@ -40,6 +43,13 @@ class GlobalPrometheusStatsWorker(Worker):
       metric_queue.org_count.Set(model.organization.get_active_org_count())
       metric_queue.robot_count.Set(model.user.get_robot_count())
 
+      if features.SECURITY_SCANNER:
+        # Clair repo counts.
+        unscanned_images_gauge.set(
+          get_count_of_images_eligible_for_scan(app.config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2))
+        )
+        images_gauge.set(total_image_count())
+
 
 def main():
   logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)

workers/securityworker.py

@@ -1,15 +1,13 @@
-import logging
 import logging.config
 import time
 
 import features
 
-from peewee import fn
-
-from app import app, secscan_api
+from app import app, secscan_api, prometheus
 from workers.worker import Worker
-from data.database import Image, UseThenDisconnect
-from data.model.image import get_image_with_storage_and_parent_base
+from data.database import UseThenDisconnect
+from data.model.image import (get_images_eligible_for_scan, get_max_id_for_sec_scan,
+                              get_min_id_for_sec_scan, get_image_id)
 from util.secscan.api import SecurityConfigValidator
 from util.secscan.analyzer import LayerAnalyzer
 from util.migrate.allocator import yield_random_entries
@@ -19,6 +17,8 @@ BATCH_SIZE = 50
 INDEXING_INTERVAL = 30
 
 logger = logging.getLogger(__name__)
+unscanned_images_gauge = prometheus.create_gauge('unscanned_images', 'Number of images that clair needs to scan.')
+images_gauge = prometheus.create_gauge('all_images', 'Total number of images that clair can scan.')
 
 class SecurityWorker(Worker):
   def __init__(self):
@@ -29,10 +29,7 @@ class SecurityWorker(Worker):
       self._analyzer = LayerAnalyzer(app.config, secscan_api)
 
       # Get the ID of the first image we want to analyze.
-      self._min_id = (Image
-                      .select(fn.Min(Image.id))
-                      .where(Image.security_indexed_engine < self._target_version)
-                      .scalar())
+      self._min_id = get_min_id_for_sec_scan(self._target_version)
 
       self.add_operation(self._index_images, INDEXING_INTERVAL)
     else:
@@ -40,17 +37,16 @@ class SecurityWorker(Worker):
 
   def _index_images(self):
     def batch_query():
-      base_query = get_image_with_storage_and_parent_base()
-      return base_query.where(Image.security_indexed_engine < self._target_version)
+      return get_images_eligible_for_scan(self._target_version)
 
     # Get the ID of the last image we can analyze. Will be None if there are no images in the
     # database.
-    max_id = Image.select(fn.Max(Image.id)).scalar()
+    max_id = get_max_id_for_sec_scan()
     if max_id is None:
       return
 
     with UseThenDisconnect(app.config):
-      for candidate, abt in yield_random_entries(batch_query, Image.id, BATCH_SIZE, max_id,
+      for candidate, abt in yield_random_entries(batch_query, get_image_id(), BATCH_SIZE, max_id,
                                                  self._min_id):
         _, continue_batch = self._analyzer.analyze_recursively(candidate)
         if not continue_batch: