From 904b902295c1e47a31f935b6ec6fbc458c6dceda Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 6 Mar 2017 14:36:53 -0500 Subject: [PATCH 1/3] workers.securityworker: find eligible tag images --- data/model/tag.py | 16 +++++++++++++++- workers/securityworker.py | 6 +++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/data/model/tag.py b/data/model/tag.py index 239bc3468..b9d66bf00 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -2,7 +2,7 @@ import logging from uuid import uuid4 -from peewee import IntegrityError +from peewee import IntegrityError, JOIN_LEFT_OUTER from data.model import (image, db_transaction, DataModelException, _basequery, InvalidManifestException, TagAlreadyCreatedException, StaleTagException) from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest, @@ -13,6 +13,20 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names logger = logging.getLogger(__name__) +def get_tags_images_eligible_for_scan(clair_version): + Parent = Image.alias() + ParentImageStorage = ImageStorage.alias() + + return _tag_alive(Image + .select(Image, ImageStorage, Parent, ParentImageStorage, RepositoryTag) + .join(RepositoryTag, on=(RepositoryTag.image == Image.id)) + .join(ImageStorage, on=(Image.storage == ImageStorage.id)) + .switch(Image) + .join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id)) + .join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)) + .where(Image.security_indexed_engine < clair_version)) + + def _tag_alive(query, now_ts=None): if now_ts is None: now_ts = get_epoch_timestamp() diff --git a/workers/securityworker.py b/workers/securityworker.py index 9c92c1f19..46b1c8e6e 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -6,8 +6,8 @@ import features from app import app, secscan_api, prometheus from workers.worker import Worker from data.database import UseThenDisconnect -from data.model.image import (get_images_eligible_for_scan, get_max_id_for_sec_scan, - get_min_id_for_sec_scan, get_image_id) +from data.model.image import get_max_id_for_sec_scan, get_min_id_for_sec_scan, get_image_id +from data.model.tag import get_tags_images_eligible_for_scan from util.secscan.api import SecurityConfigValidator from util.secscan.analyzer import LayerAnalyzer, PreemptedException from util.migrate.allocator import yield_random_entries @@ -43,7 +43,7 @@ class SecurityWorker(Worker): def _index_images(self): def batch_query(): - return get_images_eligible_for_scan(self._target_version) + return get_tags_images_eligible_for_scan(self._target_version) # Get the ID of the last image we can analyze. Will be None if there are no images in the # database. From 2cead05f53e42fc1b2a5c8cd854a721523a430ed Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 6 Mar 2017 15:44:01 -0500 Subject: [PATCH 2/3] data.model.tag: filter hidden for scan eligibility --- data/model/tag.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/model/tag.py b/data/model/tag.py index b9d66bf00..8d3584597 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -24,6 +24,7 @@ def get_tags_images_eligible_for_scan(clair_version): .switch(Image) .join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id)) .join(ParentImageStorage, JOIN_LEFT_OUTER, on=(ParentImageStorage.id == Parent.storage)) + .where(RepositoryTag.hidden == False) .where(Image.security_indexed_engine < clair_version)) From 40636d41033d9e1b2158c0ab7149a2bf23ca7879 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Mon, 6 Mar 2017 16:11:17 -0500 Subject: [PATCH 3/3] find work based on tag IDs rather than image IDs --- data/model/tag.py | 25 ++++++++++++++++++++++--- workers/securityworker.py | 8 ++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/data/model/tag.py b/data/model/tag.py index 8d3584597..98d0579fa 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -2,7 +2,7 @@ import logging from uuid import uuid4 -from peewee import IntegrityError, JOIN_LEFT_OUTER +from peewee import IntegrityError, JOIN_LEFT_OUTER, fn from data.model import (image, db_transaction, DataModelException, _basequery, InvalidManifestException, TagAlreadyCreatedException, StaleTagException) from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest, @@ -13,13 +13,32 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names logger = logging.getLogger(__name__) +def get_max_id_for_sec_scan(): + """ Gets the maximum id for security scanning """ + return RepositoryTag.select(fn.Max(RepositoryTag.id)).scalar() + + +def get_min_id_for_sec_scan(version): + """ Gets the minimum id for a security scanning """ + return (RepositoryTag + .select(fn.Min(RepositoryTag.id)) + .join(Image) + .where(Image.security_indexed_engine < version) + .scalar()) + + +def get_tag_pk_field(): + """ Returns the primary key for Image DB model """ + return RepositoryTag.id + + def get_tags_images_eligible_for_scan(clair_version): Parent = Image.alias() ParentImageStorage = ImageStorage.alias() - return _tag_alive(Image + return _tag_alive(RepositoryTag .select(Image, ImageStorage, Parent, ParentImageStorage, RepositoryTag) - .join(RepositoryTag, on=(RepositoryTag.image == Image.id)) + .join(Image, on=(RepositoryTag.image == Image.id)) .join(ImageStorage, on=(Image.storage == ImageStorage.id)) .switch(Image) .join(Parent, JOIN_LEFT_OUTER, on=(Image.parent == Parent.id)) diff --git a/workers/securityworker.py b/workers/securityworker.py index 46b1c8e6e..964d3c5b1 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -6,8 +6,8 @@ import features from app import app, secscan_api, prometheus from workers.worker import Worker from data.database import UseThenDisconnect -from data.model.image import get_max_id_for_sec_scan, get_min_id_for_sec_scan, get_image_id -from data.model.tag import get_tags_images_eligible_for_scan +from data.model.tag import (get_tags_images_eligible_for_scan, get_tag_pk_field, + get_max_id_for_sec_scan, get_min_id_for_sec_scan) from util.secscan.api import SecurityConfigValidator from util.secscan.analyzer import LayerAnalyzer, PreemptedException from util.migrate.allocator import yield_random_entries @@ -56,14 +56,14 @@ class SecurityWorker(Worker): with UseThenDisconnect(app.config): to_scan_generator = yield_random_entries( batch_query, - get_image_id(), + get_tag_pk_field(), BATCH_SIZE, max_id, self._min_id, ) for candidate, abt, num_remaining in to_scan_generator: try: - self._analyzer.analyze_recursively(candidate) + self._analyzer.analyze_recursively(candidate.image) except PreemptedException: logger.info('Another worker pre-empted us for layer: %s', candidate.id) abt.set()