vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix deleting repos when sec scan or signing is disabled

Browse source 
Make sure we don't invoke the APIs to non-existent endpoints
This commit is contained in:
Joseph Schorr 2017-04-19 13:51:13 -04:00
parent 624d8e1851
commit c5bb9abf11
4 changed files with 181 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
data/model/test/test_repository.py
View file

@ -2,7 +2,7 @@ import pytest
from peewee import IntegrityError
from data.model.repository import create_repository
from data.model.repository import create_repository, purge_repository
from test.fixtures import database_uri, init_db_path, sqlitedb_file
def test_duplicate_repository_different_kinds(database_uri):

20
util/abchelpers.py Normal file
View file

@ -0,0 +1,20 @@
class NoopIsANoopException(TypeError):
""" Raised if the nooper decorator is unnecessary on a class. """
pass
def nooper(cls):
""" Decorates a class that derives from an ABCMeta, filling in any unimplemented methods with
no-ops.
"""
def empty_func(self_or_cls, *args, **kwargs):
pass
empty_methods = {}
for method in cls.__abstractmethods__:
empty_methods[method] = empty_func
if not empty_methods:
raise NoopIsANoopException('nooper implemented no abstract methods on %s' % cls)
return type(cls.__name__, (cls,), empty_methods)

101
util/secscan/api.py
View file

@ -1,5 +1,7 @@
import logging
from abc import ABCMeta, abstractmethod
from six import add_metaclass
from urlparse import urljoin
import requests
@ -9,11 +11,12 @@ from flask import url_for
from data.database import CloseForLongOperation
from data import model
from data.model.storage import get_storage_locations
from util import get_app_url
from util.abchelpers import nooper
from util.failover import failover, FailoverException
from util.secscan.validator import SecurityConfigValidator
from util.security.instancekeys import InstanceKeys
from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
from util import get_app_url
TOKEN_VALIDITY_LIFETIME_S = 60 # Amount of time the security scanner has to call the layer URL
@ -64,14 +67,88 @@ def compute_layer_id(layer):
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (Clair). """
""" Helper class for talking to the Security Scan service (usually Clair). """
def __init__(self, app, config, storage, client=None, skip_validation=False):
if not skip_validation:
config_validator = SecurityConfigValidator(config)
if not config_validator.valid():
logger.warning('Invalid config provided to SecurityScannerAPI')
return
feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False)
has_valid_config = skip_validation
if not skip_validation and feature_enabled:
config_validator = SecurityConfigValidator(config)
has_valid_config = config_validator.valid()
if feature_enabled and has_valid_config:
self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client)
else:
self.state = NoopSecurityScannerAPI()
def __getattr__(self, name):
return getattr(self.state, name, None)
@add_metaclass(ABCMeta)
class SecurityScannerAPIInterface(object):
""" Helper class for talking to the Security Scan service (usually Clair). """
@abstractmethod
def cleanup_layers(self, layers):
""" Callback invoked by garbage collection to cleanup any layers that no longer
need to be stored in the security scanner.
"""
pass
@abstractmethod
def ping(self):
""" Calls GET on the metrics endpoint of the security scanner to ensure it is running
and properly configured. Returns the HTTP response.
"""
pass
@abstractmethod
def delete_layer(self, layer):
""" Calls DELETE on the given layer in the security scanner, removing it from
its database.
"""
pass
@abstractmethod
def analyze_layer(self, layer):
""" Posts the given layer to the security scanner for analysis, blocking until complete.
Returns the analysis version on success or raises an exception deriving from
AnalyzeLayerException on failure. Callers should handle all cases of AnalyzeLayerException.
"""
pass
@abstractmethod
def check_layer_vulnerable(self, layer_id, cve_name):
""" Checks to see if the layer with the given ID is vulnerable to the specified CVE. """
pass
@abstractmethod
def get_notification(self, notification_name, layer_limit=100, page=None):
""" Gets the data for a specific notification, with optional page token.
Returns a tuple of the data (None on failure) and whether to retry.
"""
pass
@abstractmethod
def mark_notification_read(self, notification_name):
""" Marks a security scanner notification as read. """
pass
@abstractmethod
def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
""" Returns the layer data for the specified layer. On error, returns None. """
pass
@nooper
class NoopSecurityScannerAPI(SecurityScannerAPIInterface):
""" No-op version of the security scanner API. """
pass
class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
""" Helper class for talking to the Security Scan service (Clair). """
def __init__(self, app, config, storage, client=None):
self._app = app
self._config = config
self._instance_keys = InstanceKeys(app)
@ -153,10 +230,6 @@ class SecurityScannerAPI(object):
""" Callback invoked by garbage collection to cleanup any layers that no longer
need to be stored in the security scanner.
"""
if self._config is None:
# Security scanner not enabled.
return
for layer in layers:
self.delete_layer(layer)
@ -339,9 +412,6 @@ class SecurityScannerAPI(object):
def _request(self, method, endpoint, path, body, params, timeout):
""" Issues an HTTP request to the security endpoint. """
if self._config is None:
raise Exception('Cannot call unconfigured security system')
url = _join_api_url(endpoint, self._config.get('SECURITY_SCANNER_API_VERSION', 'v1'), path)
signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080')
@ -358,9 +428,6 @@ class SecurityScannerAPI(object):
""" Issues an HTTP request to the security endpoint handling the logic of using an alternative
BATCH endpoint for non-GET requests and failover for GET requests.
"""
if self._config is None:
raise Exception('Cannot call unconfigured security system')
timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 1)
endpoint = self._config['SECURITY_SCANNER_ENDPOINT']

98
util/tufmetadata/api.py
View file

@ -1,10 +1,14 @@
import logging
from abc import ABCMeta, abstractmethod
from six import add_metaclass
from urlparse import urljoin
from posixpath import join
import requests
from data.database import CloseForLongOperation
from util.abchelpers import nooper
from util.failover import failover, FailoverException
from util.security.instancekeys import InstanceKeys
from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT
@ -30,7 +34,60 @@ class Non200ResponseException(Exception):
class TUFMetadataAPI(object):
""" Helper class for talking to the Security Scan service (usually Clair). """
def __init__(self, app, config, client=None):
feature_enabled = config.get('FEATURE_SIGNING', False)
if feature_enabled:
self.state = ImplementedTUFMetadataAPI(app, config, client=client)
else:
self.state = NoopTUFMetadataAPI()
def __getattr__(self, name):
return getattr(self.state, name, None)
@add_metaclass(ABCMeta)
class TUFMetadataAPIInterface(object):
""" Helper class for talking to the TUF Metadata service (Apostille). """
@abstractmethod
def get_default_tags_with_expiration(self, namespace, repository, targets_file=None):
"""
Gets the tag -> sha mappings for a repo, as well as the expiration of the signatures.
Does not verify the metadata, this is purely for display purposes.
Args:
namespace: namespace containing the repository
repository: the repo to get tags for
targets_file: the specific delegation to read from. Default: targets/releases.json
Returns:
targets, expiration or None, None
"""
pass
@abstractmethod
def delete_metadata(self, namespace, repository):
"""
Deletes the TUF metadata for a repo
Args:
namespace: namespace containing the repository
repository: the repo to delete metadata for
Returns:
True if successful, False otherwise
"""
pass
@nooper
class NoopTUFMetadataAPI(TUFMetadataAPIInterface):
""" No-op version of the TUF API. """
pass
class ImplementedTUFMetadataAPI(TUFMetadataAPIInterface):
def __init__(self, app, config, client=None):
self._app = app
self._instance_keys = InstanceKeys(app)
@ -42,7 +99,7 @@ class TUFMetadataAPI(object):
"""
Gets the tag -> sha mappings for a repo, as well as the expiration of the signatures.
Does not verify the metadata, this is purely for display purposes.
Args:
namespace: namespace containing the repository
repository: the repo to get tags for
@ -51,11 +108,11 @@ class TUFMetadataAPI(object):
Returns:
targets, expiration or None, None
"""
if not targets_file:
targets_file = 'targets/releases.json'
gun = self._gun(namespace, repository)
try:
response = self._get(gun, targets_file)
signed = self._parse_signed(response.json())
@ -63,26 +120,26 @@ class TUFMetadataAPI(object):
expiration = signed.get('expires')
except requests.exceptions.Timeout:
logger.exception('Timeout when trying to get metadata for %s', gun)
return None, None
return None, None
except requests.exceptions.ConnectionError:
logger.exception('Connection error when trying to get metadata for %s', gun)
return None, None
return None, None
except (requests.exceptions.RequestException, ValueError):
logger.exception('Failed to get metadata for %s', gun)
return None, None
return None, None
except Non200ResponseException as ex:
logger.exception('Failed request for %s: %s', gun, str(ex))
return None, None
return None, None
except InvalidMetadataException as ex:
logger.exception('Failed to parse targets from metadata', str(ex))
return None, None
return None, None
return targets, expiration
def delete_metadata(self, namespace, repository):
"""
Deletes the TUF metadata for a repo
Args:
namespace: namespace containing the repository
repository: the repo to delete metadata for
@ -92,7 +149,7 @@ class TUFMetadataAPI(object):
"""
gun = self._gun(namespace, repository)
try:
self._delete(gun)
self._delete(gun)
except requests.exceptions.Timeout:
logger.exception('Timeout when trying to delete metadata for %s', gun)
return False
@ -106,17 +163,17 @@ class TUFMetadataAPI(object):
logger.exception('Failed request for %s: %s', gun, str(ex))
return False
return True
def _gun(self, namespace, repository):
return join(self._gun_prefix, namespace, repository)
def _parse_signed(self, json_response):
""" Attempts to parse the targets from a metadata response """
signed = json_response.get('signed')
if not signed:
raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response)
return signed
return signed
def _auth_header(self, gun, actions):
""" Generate a registry auth token for apostille"""
access = [{
@ -128,19 +185,19 @@ class TUFMetadataAPI(object):
token = generate_bearer_token(self._config["SERVER_HOSTNAME"], subject, context, access,
TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
return {'Authorization': 'Bearer %s' % token}
def _get(self, gun, metadata_file):
return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun, ['pull']))
def _delete(self, gun):
return self._call('DELETE', '/v2/%s/_trust/tuf/' % (gun), headers=self._auth_header(gun, ['*']))
def _request(self, method, endpoint, path, body, headers, params, timeout):
""" Issues an HTTP request to the signing endpoint. """
url = urljoin(endpoint, path)
logger.debug('%sing signing URL %s', method.upper(), url)
headers.update(DEFAULT_HTTP_HEADERS)
headers.update(DEFAULT_HTTP_HEADERS)
resp = self._client.request(method, url, json=body, params=params, timeout=timeout,
verify=MITM_CERT_PATH, headers=headers)
if resp.status_code // 100 != 2:
@ -150,9 +207,6 @@ class TUFMetadataAPI(object):
def _call(self, method, path, params=None, body=None, headers=None):
""" Issues an HTTP request to signing service and handles failover for GET requests.
"""
if self._config is None:
raise Exception('Cannot call unconfigured signing service')
timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1)
endpoint = self._config['TUF_SERVER']