From c5bb9abf11dc9a3f998ea21d9fe4a9004a1bdbf4 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 19 Apr 2017 13:51:13 -0400 Subject: [PATCH] Fix deleting repos when sec scan or signing is disabled Make sure we don't invoke the APIs to non-existent endpoints --- data/model/test/test_repository.py | 2 +- util/abchelpers.py | 20 ++++++ util/secscan/api.py | 101 ++++++++++++++++++++++++----- util/tufmetadata/api.py | 98 +++++++++++++++++++++------- 4 files changed, 181 insertions(+), 40 deletions(-) create mode 100644 util/abchelpers.py diff --git a/data/model/test/test_repository.py b/data/model/test/test_repository.py index 76308142e..f0265a341 100644 --- a/data/model/test/test_repository.py +++ b/data/model/test/test_repository.py @@ -2,7 +2,7 @@ import pytest from peewee import IntegrityError -from data.model.repository import create_repository +from data.model.repository import create_repository, purge_repository from test.fixtures import database_uri, init_db_path, sqlitedb_file def test_duplicate_repository_different_kinds(database_uri): diff --git a/util/abchelpers.py b/util/abchelpers.py new file mode 100644 index 000000000..8908538a2 --- /dev/null +++ b/util/abchelpers.py @@ -0,0 +1,20 @@ +class NoopIsANoopException(TypeError): + """ Raised if the nooper decorator is unnecessary on a class. """ + pass + + +def nooper(cls): + """ Decorates a class that derives from an ABCMeta, filling in any unimplemented methods with + no-ops. + """ + def empty_func(self_or_cls, *args, **kwargs): + pass + + empty_methods = {} + for method in cls.__abstractmethods__: + empty_methods[method] = empty_func + + if not empty_methods: + raise NoopIsANoopException('nooper implemented no abstract methods on %s' % cls) + + return type(cls.__name__, (cls,), empty_methods) diff --git a/util/secscan/api.py b/util/secscan/api.py index 7c080468d..4dec14a7f 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -1,5 +1,7 @@ import logging +from abc import ABCMeta, abstractmethod +from six import add_metaclass from urlparse import urljoin import requests @@ -9,11 +11,12 @@ from flask import url_for from data.database import CloseForLongOperation from data import model from data.model.storage import get_storage_locations +from util import get_app_url +from util.abchelpers import nooper from util.failover import failover, FailoverException from util.secscan.validator import SecurityConfigValidator from util.security.instancekeys import InstanceKeys from util.security.registry_jwt import generate_bearer_token, build_context_and_subject -from util import get_app_url TOKEN_VALIDITY_LIFETIME_S = 60 # Amount of time the security scanner has to call the layer URL @@ -64,14 +67,88 @@ def compute_layer_id(layer): class SecurityScannerAPI(object): - """ Helper class for talking to the Security Scan service (Clair). """ + """ Helper class for talking to the Security Scan service (usually Clair). """ def __init__(self, app, config, storage, client=None, skip_validation=False): - if not skip_validation: - config_validator = SecurityConfigValidator(config) - if not config_validator.valid(): - logger.warning('Invalid config provided to SecurityScannerAPI') - return + feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False) + has_valid_config = skip_validation + if not skip_validation and feature_enabled: + config_validator = SecurityConfigValidator(config) + has_valid_config = config_validator.valid() + + if feature_enabled and has_valid_config: + self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client) + else: + self.state = NoopSecurityScannerAPI() + + def __getattr__(self, name): + return getattr(self.state, name, None) + + +@add_metaclass(ABCMeta) +class SecurityScannerAPIInterface(object): + """ Helper class for talking to the Security Scan service (usually Clair). """ + @abstractmethod + def cleanup_layers(self, layers): + """ Callback invoked by garbage collection to cleanup any layers that no longer + need to be stored in the security scanner. + """ + pass + + @abstractmethod + def ping(self): + """ Calls GET on the metrics endpoint of the security scanner to ensure it is running + and properly configured. Returns the HTTP response. + """ + pass + + @abstractmethod + def delete_layer(self, layer): + """ Calls DELETE on the given layer in the security scanner, removing it from + its database. + """ + pass + + @abstractmethod + def analyze_layer(self, layer): + """ Posts the given layer to the security scanner for analysis, blocking until complete. + Returns the analysis version on success or raises an exception deriving from + AnalyzeLayerException on failure. Callers should handle all cases of AnalyzeLayerException. + """ + pass + + @abstractmethod + def check_layer_vulnerable(self, layer_id, cve_name): + """ Checks to see if the layer with the given ID is vulnerable to the specified CVE. """ + pass + + @abstractmethod + def get_notification(self, notification_name, layer_limit=100, page=None): + """ Gets the data for a specific notification, with optional page token. + Returns a tuple of the data (None on failure) and whether to retry. + """ + pass + + @abstractmethod + def mark_notification_read(self, notification_name): + """ Marks a security scanner notification as read. """ + pass + + @abstractmethod + def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False): + """ Returns the layer data for the specified layer. On error, returns None. """ + pass + + +@nooper +class NoopSecurityScannerAPI(SecurityScannerAPIInterface): + """ No-op version of the security scanner API. """ + pass + + +class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): + """ Helper class for talking to the Security Scan service (Clair). """ + def __init__(self, app, config, storage, client=None): self._app = app self._config = config self._instance_keys = InstanceKeys(app) @@ -153,10 +230,6 @@ class SecurityScannerAPI(object): """ Callback invoked by garbage collection to cleanup any layers that no longer need to be stored in the security scanner. """ - if self._config is None: - # Security scanner not enabled. - return - for layer in layers: self.delete_layer(layer) @@ -339,9 +412,6 @@ class SecurityScannerAPI(object): def _request(self, method, endpoint, path, body, params, timeout): """ Issues an HTTP request to the security endpoint. """ - if self._config is None: - raise Exception('Cannot call unconfigured security system') - url = _join_api_url(endpoint, self._config.get('SECURITY_SCANNER_API_VERSION', 'v1'), path) signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080') @@ -358,9 +428,6 @@ class SecurityScannerAPI(object): """ Issues an HTTP request to the security endpoint handling the logic of using an alternative BATCH endpoint for non-GET requests and failover for GET requests. """ - if self._config is None: - raise Exception('Cannot call unconfigured security system') - timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 1) endpoint = self._config['SECURITY_SCANNER_ENDPOINT'] diff --git a/util/tufmetadata/api.py b/util/tufmetadata/api.py index f3520ca4b..e3ee34147 100644 --- a/util/tufmetadata/api.py +++ b/util/tufmetadata/api.py @@ -1,10 +1,14 @@ import logging + +from abc import ABCMeta, abstractmethod +from six import add_metaclass from urlparse import urljoin from posixpath import join import requests from data.database import CloseForLongOperation +from util.abchelpers import nooper from util.failover import failover, FailoverException from util.security.instancekeys import InstanceKeys from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT @@ -30,7 +34,60 @@ class Non200ResponseException(Exception): class TUFMetadataAPI(object): + """ Helper class for talking to the Security Scan service (usually Clair). """ + def __init__(self, app, config, client=None): + feature_enabled = config.get('FEATURE_SIGNING', False) + if feature_enabled: + self.state = ImplementedTUFMetadataAPI(app, config, client=client) + else: + self.state = NoopTUFMetadataAPI() + + def __getattr__(self, name): + return getattr(self.state, name, None) + + +@add_metaclass(ABCMeta) +class TUFMetadataAPIInterface(object): """ Helper class for talking to the TUF Metadata service (Apostille). """ + + @abstractmethod + def get_default_tags_with_expiration(self, namespace, repository, targets_file=None): + """ + Gets the tag -> sha mappings for a repo, as well as the expiration of the signatures. + Does not verify the metadata, this is purely for display purposes. + + Args: + namespace: namespace containing the repository + repository: the repo to get tags for + targets_file: the specific delegation to read from. Default: targets/releases.json + + Returns: + targets, expiration or None, None + """ + pass + + @abstractmethod + def delete_metadata(self, namespace, repository): + """ + Deletes the TUF metadata for a repo + + Args: + namespace: namespace containing the repository + repository: the repo to delete metadata for + + Returns: + True if successful, False otherwise + """ + pass + + +@nooper +class NoopTUFMetadataAPI(TUFMetadataAPIInterface): + """ No-op version of the TUF API. """ + pass + + +class ImplementedTUFMetadataAPI(TUFMetadataAPIInterface): def __init__(self, app, config, client=None): self._app = app self._instance_keys = InstanceKeys(app) @@ -42,7 +99,7 @@ class TUFMetadataAPI(object): """ Gets the tag -> sha mappings for a repo, as well as the expiration of the signatures. Does not verify the metadata, this is purely for display purposes. - + Args: namespace: namespace containing the repository repository: the repo to get tags for @@ -51,11 +108,11 @@ class TUFMetadataAPI(object): Returns: targets, expiration or None, None """ - + if not targets_file: targets_file = 'targets/releases.json' gun = self._gun(namespace, repository) - + try: response = self._get(gun, targets_file) signed = self._parse_signed(response.json()) @@ -63,26 +120,26 @@ class TUFMetadataAPI(object): expiration = signed.get('expires') except requests.exceptions.Timeout: logger.exception('Timeout when trying to get metadata for %s', gun) - return None, None + return None, None except requests.exceptions.ConnectionError: logger.exception('Connection error when trying to get metadata for %s', gun) - return None, None + return None, None except (requests.exceptions.RequestException, ValueError): logger.exception('Failed to get metadata for %s', gun) - return None, None + return None, None except Non200ResponseException as ex: logger.exception('Failed request for %s: %s', gun, str(ex)) - return None, None + return None, None except InvalidMetadataException as ex: logger.exception('Failed to parse targets from metadata', str(ex)) - return None, None - + return None, None + return targets, expiration - + def delete_metadata(self, namespace, repository): """ Deletes the TUF metadata for a repo - + Args: namespace: namespace containing the repository repository: the repo to delete metadata for @@ -92,7 +149,7 @@ class TUFMetadataAPI(object): """ gun = self._gun(namespace, repository) try: - self._delete(gun) + self._delete(gun) except requests.exceptions.Timeout: logger.exception('Timeout when trying to delete metadata for %s', gun) return False @@ -106,17 +163,17 @@ class TUFMetadataAPI(object): logger.exception('Failed request for %s: %s', gun, str(ex)) return False return True - + def _gun(self, namespace, repository): return join(self._gun_prefix, namespace, repository) - + def _parse_signed(self, json_response): """ Attempts to parse the targets from a metadata response """ signed = json_response.get('signed') if not signed: raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response) - return signed - + return signed + def _auth_header(self, gun, actions): """ Generate a registry auth token for apostille""" access = [{ @@ -128,19 +185,19 @@ class TUFMetadataAPI(object): token = generate_bearer_token(self._config["SERVER_HOSTNAME"], subject, context, access, TOKEN_VALIDITY_LIFETIME_S, self._instance_keys) return {'Authorization': 'Bearer %s' % token} - + def _get(self, gun, metadata_file): return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun, ['pull'])) def _delete(self, gun): return self._call('DELETE', '/v2/%s/_trust/tuf/' % (gun), headers=self._auth_header(gun, ['*'])) - + def _request(self, method, endpoint, path, body, headers, params, timeout): """ Issues an HTTP request to the signing endpoint. """ url = urljoin(endpoint, path) logger.debug('%sing signing URL %s', method.upper(), url) - headers.update(DEFAULT_HTTP_HEADERS) + headers.update(DEFAULT_HTTP_HEADERS) resp = self._client.request(method, url, json=body, params=params, timeout=timeout, verify=MITM_CERT_PATH, headers=headers) if resp.status_code // 100 != 2: @@ -150,9 +207,6 @@ class TUFMetadataAPI(object): def _call(self, method, path, params=None, body=None, headers=None): """ Issues an HTTP request to signing service and handles failover for GET requests. """ - if self._config is None: - raise Exception('Cannot call unconfigured signing service') - timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1) endpoint = self._config['TUF_SERVER']