From 46edebe6b0ab53ba635bef64df1b8b108c73343e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Aug 2018 16:36:04 -0400 Subject: [PATCH] Change secscan API endpoints to use new registry model interface --- data/registry_model/datatypes.py | 10 +++ data/registry_model/interface.py | 4 ++ data/registry_model/registry_pre_oci_model.py | 24 ++++++- .../registry_model/test/test_pre_oci_model.py | 9 +++ endpoints/api/secscan.py | 62 ++++++++----------- util/secscan/api.py | 9 ++- 6 files changed, 81 insertions(+), 37 deletions(-) diff --git a/data/registry_model/datatypes.py b/data/registry_model/datatypes.py index dd229f6cf..89a9ad2f2 100644 --- a/data/registry_model/datatypes.py +++ b/data/registry_model/datatypes.py @@ -1,3 +1,5 @@ +from enum import Enum, unique + from data.registry_model.datatype import datatype, requiresinput class RepositoryReference(datatype('Repository', [])): @@ -107,3 +109,11 @@ class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'commen return [] return [Tag.for_repository_tag(tag) for tag in tags] + + +@unique +class SecurityScanStatus(Enum): + """ Security scan status enum """ + SCANNED = 'scanned' + FAILED = 'failed' + QUEUED = 'queued' diff --git a/data/registry_model/interface.py b/data/registry_model/interface.py index bedbb02ce..323d83f85 100644 --- a/data/registry_model/interface.py +++ b/data/registry_model/interface.py @@ -118,3 +118,7 @@ class RegistryDataInterface(object): @abstractmethod def get_legacy_images_owned_by_tag(self, tag): """ Returns all legacy images *solely owned and used* by the given tag. """ + + @abstractmethod + def get_security_status(self, manifest_or_legacy_image): + """ Returns the security status for the given manifest or legacy image or None if none. """ diff --git a/data/registry_model/registry_pre_oci_model.py b/data/registry_model/registry_pre_oci_model.py index beb9c5866..a7f5eabfe 100644 --- a/data/registry_model/registry_pre_oci_model.py +++ b/data/registry_model/registry_pre_oci_model.py @@ -5,7 +5,8 @@ from collections import defaultdict from data import database from data import model from data.registry_model.interface import RegistryDataInterface -from data.registry_model.datatypes import Tag, RepositoryReference, Manifest, LegacyImage, Label +from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label, + SecurityScanStatus) class PreOCIModel(RegistryDataInterface): @@ -267,5 +268,26 @@ class PreOCIModel(RegistryDataInterface): return [LegacyImage.for_image(image, images_map=images_map) for image in images] + def get_security_status(self, manifest_or_legacy_image): + """ Returns the security status for the given manifest or legacy image or None if none. """ + image = None + + if isinstance(manifest_or_legacy_image, Manifest): + try: + tag_manifest = database.TagManifest.get(id=manifest_or_legacy_image._db_id) + image = tag_manifest.tag.image + except database.TagManifest.DoesNotExist: + return None + else: + try: + image = database.Image.get(id=manifest_or_legacy_image._db_id) + except database.Image.DoesNotExist: + return None + + if image.security_indexed_engine is not None and image.security_indexed_engine >= 0: + return SecurityScanStatus.SCANNED if image.security_indexed else SecurityScanStatus.FAILED + + return SecurityScanStatus.QUEUED + pre_oci_model = PreOCIModel() diff --git a/data/registry_model/test/test_pre_oci_model.py b/data/registry_model/test/test_pre_oci_model.py index 50c1a36f9..a5b203c66 100644 --- a/data/registry_model/test/test_pre_oci_model.py +++ b/data/registry_model/test/test_pre_oci_model.py @@ -293,3 +293,12 @@ def test_get_legacy_images_owned_by_tag(repo_namespace, repo_name, expected_non_ non_empty.add(tag.name) assert non_empty == set(expected_non_empty) + + +def test_get_security_status(pre_oci_model): + repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') + tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True) + assert len(tags) + + for tag in tags: + assert pre_oci_model.get_security_status(tag.legacy_image) diff --git a/endpoints/api/secscan.py b/endpoints/api/secscan.py index 3fd04a66d..a5ae88043 100644 --- a/endpoints/api/secscan.py +++ b/endpoints/api/secscan.py @@ -4,7 +4,8 @@ import logging import features from app import secscan_api -from data import model +from data.registry_model import registry_model +from data.registry_model.datatypes import SecurityScanStatus from endpoints.api import (require_repo_read, path_param, RepositoryParamResource, resource, nickname, show_if, parse_args, query_param, truthy_bool, disallow_for_app_repositories) @@ -15,37 +16,24 @@ from util.secscan.api import APIRequestFailure logger = logging.getLogger(__name__) - -class SCAN_STATUS(object): - """ Security scan status enum """ - SCANNED = 'scanned' - FAILED = 'failed' - QUEUED = 'queued' - - -def _get_status(repo_image): - """ Returns the SCAN_STATUS value for the given image. """ - if repo_image.security_indexed_engine is not None and repo_image.security_indexed_engine >= 0: - return SCAN_STATUS.SCANNED if repo_image.security_indexed else SCAN_STATUS.FAILED - - return SCAN_STATUS.QUEUED - -def _security_status_for_image(namespace, repository, repo_image, include_vulnerabilities=True): +def _security_info(manifest_or_legacy_image, include_vulnerabilities=True): """ Returns a dict representing the result of a call to the security status API for the given - image. + manifest or image. """ - if not repo_image.security_indexed: - logger.debug('Image %s under repository %s/%s not security indexed', - repo_image.docker_image_id, namespace, repository) + status = registry_model.get_security_status(manifest_or_legacy_image) + if status is None: + raise NotFound() + + if status != SecurityScanStatus.SCANNED: return { - 'status': _get_status(repo_image), + 'status': status.value, } try: if include_vulnerabilities: - data = secscan_api.get_layer_data(repo_image, include_vulnerabilities=True) + data = secscan_api.get_layer_data(manifest_or_legacy_image, include_vulnerabilities=True) else: - data = secscan_api.get_layer_data(repo_image, include_features=True) + data = secscan_api.get_layer_data(manifest_or_legacy_image, include_features=True) except APIRequestFailure as arf: raise DownstreamIssue(arf.message) @@ -53,7 +41,7 @@ def _security_status_for_image(namespace, repository, repo_image, include_vulner raise NotFound() return { - 'status': _get_status(repo_image), + 'status': status.value, 'data': data, } @@ -73,12 +61,16 @@ class RepositoryImageSecurity(RepositoryParamResource): default=False) def get(self, namespace, repository, imageid, parsed_args): """ Fetches the features and vulnerabilities (if any) for a repository image. """ - repo_image = model.image.get_repo_image(namespace, repository, imageid) - if repo_image is None: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() - return _security_status_for_image(namespace, repository, repo_image, - parsed_args.vulnerabilities) + legacy_image = registry_model.get_legacy_image(repo_ref, imageid) + if legacy_image is None: + raise NotFound() + + return _security_info(legacy_image, parsed_args.vulnerabilities) + @resource(MANIFEST_DIGEST_ROUTE + '/security') @show_if(features.SECURITY_SCANNER) @@ -94,12 +86,12 @@ class RepositoryManifestSecurity(RepositoryParamResource): @query_param('vulnerabilities', 'Include vulnerabilities informations', type=truthy_bool, default=False) def get(self, namespace, repository, manifestref, parsed_args): - try: - tag_manifest = model.tag.load_manifest_by_digest(namespace, repository, manifestref) - except model.DataModelException: + repo_ref = registry_model.lookup_repository(namespace, repository) + if repo_ref is None: raise NotFound() - repo_image = tag_manifest.tag.image + manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref, allow_dead=True) + if manifest is None: + raise NotFound() - return _security_status_for_image(namespace, repository, repo_image, - parsed_args.vulnerabilities) + return _security_info(manifest, parsed_args.vulnerabilities) diff --git a/util/secscan/api.py b/util/secscan/api.py index 25ef36807..cc6e282f2 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -7,9 +7,10 @@ from urlparse import urljoin import requests -from data.database import CloseForLongOperation from data import model +from data.database import CloseForLongOperation, TagManifest, Image from data.model.storage import get_storage_locations +from data.registry_model.datatypes import Manifest, LegacyImage from util.abchelpers import nooper from util.failover import failover, FailoverException from util.secscan.validator import SecurityConfigValidator @@ -61,6 +62,12 @@ _API_METHOD_PING = 'metrics' def compute_layer_id(layer): """ Returns the ID for the layer in the security scanner. """ + # NOTE: this is temporary until we switch to Clair V3. + if isinstance(layer, Manifest): + layer = TagManifest.get(id=layer._db_id).tag.image + elif isinstance(layer, LegacyImage): + layer = Image.get(id=layer._db_id) + return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)