From add6b654ae607f374fbbbf28424c219a550eab58 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 22 Feb 2017 11:24:41 -0500 Subject: [PATCH 1/2] Move the total image count stat back to the prom stat worker --- data/model/image.py | 8 -------- workers/globalpromstats.py | 6 ++++-- workers/securityworker.py | 4 +--- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/data/model/image.py b/data/model/image.py index 6b8a38a09..8d2d06166 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -500,14 +500,6 @@ def get_images_eligible_for_scan(clair_version): return get_image_with_storage_and_parent_base().where(Image.security_indexed_engine < clair_version) -def get_count_of_images_eligible_for_scan(clair_version): - """ Returns a query that gives all images eligible for a clair scan """ - # TODO(jzelinskie): Get this value from the slab allocator rather than querying the db - # This was the previous implementation: - # return get_images_eligible_for_scan(clair_version).count() - return 0 - - def get_image_with_storage_and_parent_base(): Parent = Image.alias() ParentImageStorage = ImageStorage.alias() diff --git a/workers/globalpromstats.py b/workers/globalpromstats.py index 199d0373f..4e98cba86 100644 --- a/workers/globalpromstats.py +++ b/workers/globalpromstats.py @@ -5,9 +5,9 @@ import features from app import app, metric_queue from data.database import UseThenDisconnect from data import model -from data.model.image import total_image_count, get_count_of_images_eligible_for_scan +from data.model.image import total_image_count from util.locking import GlobalLock, LockNotAcquiredException -from workers.securityworker import unscanned_images_gauge, images_gauge +from workers.securityworker import images_gauge from workers.worker import Worker logger = logging.getLogger(__name__) @@ -43,6 +43,8 @@ class GlobalPrometheusStatsWorker(Worker): metric_queue.org_count.Set(model.organization.get_active_org_count()) metric_queue.robot_count.Set(model.user.get_robot_count()) + images_gauge.Set(total_image_count()) + def main(): logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) diff --git a/workers/securityworker.py b/workers/securityworker.py index ebdf6d5d2..06e292ef3 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -7,7 +7,7 @@ from app import app, secscan_api, prometheus from workers.worker import Worker from data.database import UseThenDisconnect from data.model.image import (get_images_eligible_for_scan, get_max_id_for_sec_scan, - get_min_id_for_sec_scan, get_image_id, total_image_count) + get_min_id_for_sec_scan, get_image_id) from util.secscan.api import SecurityConfigValidator from util.secscan.analyzer import LayerAnalyzer, PreemptedException from util.migrate.allocator import yield_random_entries @@ -64,8 +64,6 @@ class SecurityWorker(Worker): abt.set() unscanned_images_gauge.Set(num_remaining) - images_gauge.Set(total_image_count()) - # If we reach this point, we analyzed every images up to max_id, next time the worker runs, # we want to start from the next image. From 27f5f14f900d2257cf80ff12509aa998dded7c9c Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 22 Feb 2017 11:25:09 -0500 Subject: [PATCH 2/2] Linter fixes --- data/model/image.py | 11 ++++++----- workers/securityworker.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/data/model/image.py b/data/model/image.py index 8d2d06166..cd207c468 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -1,16 +1,16 @@ import logging -import dateutil.parser import hashlib import json +import dateutil.parser -from peewee import JOIN_LEFT_OUTER, IntegrityError, fn from datetime import datetime +from peewee import JOIN_LEFT_OUTER, IntegrityError, fn from data.model import (DataModelException, db_transaction, _basequery, storage, - InvalidImageException, config) + InvalidImageException) from data.database import (Image, Repository, ImageStoragePlacement, Namespace, ImageStorage, ImageStorageLocation, RepositoryPermission, DerivedStorageForImage, - ImageStorageTransformation, db_random_func) + ImageStorageTransformation) from util.canonicaljson import canonicalize @@ -497,7 +497,8 @@ def get_image_id(): def get_images_eligible_for_scan(clair_version): """ Returns a query that gives all images eligible for a clair scan """ - return get_image_with_storage_and_parent_base().where(Image.security_indexed_engine < clair_version) + return (get_image_with_storage_and_parent_base() + .where(Image.security_indexed_engine < clair_version)) def get_image_with_storage_and_parent_base(): diff --git a/workers/securityworker.py b/workers/securityworker.py index 06e292ef3..ae093f0de 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -19,7 +19,8 @@ INDEXING_INTERVAL = 30 logger = logging.getLogger(__name__) -unscanned_images_gauge = prometheus.create_gauge('unscanned_images', 'Number of images that clair needs to scan.') +unscanned_images_gauge = prometheus.create_gauge('unscanned_images', + 'Number of images that clair needs to scan.') images_gauge = prometheus.create_gauge('all_images', 'Total number of images that clair can scan.')