Merge pull request #2554 from coreos-inc/no-secscan-delete

Fix deleting repos when sec scan or signing is disabled

data/model/test/test_repository.py

@@ -2,7 +2,7 @@ import pytest
 
 from peewee import IntegrityError
 
-from data.model.repository import create_repository
+from data.model.repository import create_repository, purge_repository
 from test.fixtures import database_uri, init_db_path, sqlitedb_file
 
 def test_duplicate_repository_different_kinds(database_uri):

util/abchelpers.py

@@ -0,0 +1,20 @@
+class NoopIsANoopException(TypeError):
+  """ Raised if the nooper decorator is unnecessary on a class. """
+  pass
+
+
+def nooper(cls):
+  """ Decorates a class that derives from an ABCMeta, filling in any unimplemented methods with
+      no-ops.
+  """
+  def empty_func(self_or_cls, *args, **kwargs):
+    pass
+
+  empty_methods = {}
+  for method in cls.__abstractmethods__:
+    empty_methods[method] = empty_func
+
+  if not empty_methods:
+    raise NoopIsANoopException('nooper implemented no abstract methods on %s' % cls)
+
+  return type(cls.__name__, (cls,), empty_methods)

util/secscan/api.py

@@ -1,5 +1,7 @@
 import logging
 
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 from urlparse import urljoin
 
 import requests
@@ -9,11 +11,12 @@ from flask import url_for
 from data.database import CloseForLongOperation
 from data import model
 from data.model.storage import get_storage_locations
+from util import get_app_url
+from util.abchelpers import nooper
 from util.failover import failover, FailoverException
 from util.secscan.validator import SecurityConfigValidator
 from util.security.instancekeys import InstanceKeys
 from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
-from util import get_app_url
 
 
 TOKEN_VALIDITY_LIFETIME_S = 60  # Amount of time the security scanner has to call the layer URL
@@ -64,14 +67,88 @@ def compute_layer_id(layer):
 
 
 class SecurityScannerAPI(object):
-  """ Helper class for talking to the Security Scan service (Clair). """
+  """ Helper class for talking to the Security Scan service (usually Clair). """
   def __init__(self, app, config, storage, client=None, skip_validation=False):
-    if not skip_validation:
-      config_validator = SecurityConfigValidator(config)
-      if not config_validator.valid():
-        logger.warning('Invalid config provided to SecurityScannerAPI')
-        return
+    feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False)
+    has_valid_config = skip_validation
 
+    if not skip_validation and feature_enabled:
+      config_validator = SecurityConfigValidator(config)
+      has_valid_config = config_validator.valid()
+
+    if feature_enabled and has_valid_config:
+      self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client)
+    else:
+      self.state = NoopSecurityScannerAPI()
+
+  def __getattr__(self, name):
+    return getattr(self.state, name, None)
+
+
+@add_metaclass(ABCMeta)
+class SecurityScannerAPIInterface(object):
+  """ Helper class for talking to the Security Scan service (usually Clair). """
+  @abstractmethod
+  def cleanup_layers(self, layers):
+    """ Callback invoked by garbage collection to cleanup any layers that no longer
+        need to be stored in the security scanner.
+    """
+    pass
+
+  @abstractmethod
+  def ping(self):
+    """ Calls GET on the metrics endpoint of the security scanner to ensure it is running
+        and properly configured. Returns the HTTP response.
+    """
+    pass
+
+  @abstractmethod
+  def delete_layer(self, layer):
+    """ Calls DELETE on the given layer in the security scanner, removing it from
+        its database.
+    """
+    pass
+
+  @abstractmethod
+  def analyze_layer(self, layer):
+    """ Posts the given layer to the security scanner for analysis, blocking until complete.
+        Returns the analysis version on success or raises an exception deriving from
+        AnalyzeLayerException on failure. Callers should handle all cases of AnalyzeLayerException.
+    """
+    pass
+
+  @abstractmethod
+  def check_layer_vulnerable(self, layer_id, cve_name):
+    """ Checks to see if the layer with the given ID is vulnerable to the specified CVE. """
+    pass
+
+  @abstractmethod
+  def get_notification(self, notification_name, layer_limit=100, page=None):
+    """ Gets the data for a specific notification, with optional page token.
+        Returns a tuple of the data (None on failure) and whether to retry.
+    """
+    pass
+
+  @abstractmethod
+  def mark_notification_read(self, notification_name):
+    """ Marks a security scanner notification as read. """
+    pass
+
+  @abstractmethod
+  def get_layer_data(self, layer, include_features=False, include_vulnerabilities=False):
+    """ Returns the layer data for the specified layer. On error, returns None. """
+    pass
+
+
+@nooper
+class NoopSecurityScannerAPI(SecurityScannerAPIInterface):
+  """ No-op version of the security scanner API. """
+  pass
+
+
+class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
+  """ Helper class for talking to the Security Scan service (Clair). """
+  def __init__(self, app, config, storage, client=None):
     self._app = app
     self._config = config
     self._instance_keys = InstanceKeys(app)
@@ -153,10 +230,6 @@ class SecurityScannerAPI(object):
     """ Callback invoked by garbage collection to cleanup any layers that no longer
         need to be stored in the security scanner.
     """
-    if self._config is None:
-      # Security scanner not enabled.
-      return
-
     for layer in layers:
       self.delete_layer(layer)
 
@@ -339,9 +412,6 @@ class SecurityScannerAPI(object):
 
   def _request(self, method, endpoint, path, body, params, timeout):
     """ Issues an HTTP request to the security endpoint. """
-    if self._config is None:
-      raise Exception('Cannot call unconfigured security system')
-
     url = _join_api_url(endpoint, self._config.get('SECURITY_SCANNER_API_VERSION', 'v1'), path)
     signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080')
 
@@ -358,9 +428,6 @@ class SecurityScannerAPI(object):
     """ Issues an HTTP request to the security endpoint handling the logic of using an alternative
         BATCH endpoint for non-GET requests and failover for GET requests.
     """
-    if self._config is None:
-      raise Exception('Cannot call unconfigured security system')
-
     timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 1)
     endpoint = self._config['SECURITY_SCANNER_ENDPOINT']
 

util/tufmetadata/api.py

@@ -1,10 +1,14 @@
 import logging
+
+from abc import ABCMeta, abstractmethod
+from six import add_metaclass
 from urlparse import urljoin
 from posixpath import join
 
 import requests
 
 from data.database import CloseForLongOperation
+from util.abchelpers import nooper
 from util.failover import failover, FailoverException
 from util.security.instancekeys import InstanceKeys
 from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT
@@ -30,7 +34,60 @@ class Non200ResponseException(Exception):
 
 
 class TUFMetadataAPI(object):
+  """ Helper class for talking to the Security Scan service (usually Clair). """
+  def __init__(self, app, config, client=None):
+    feature_enabled = config.get('FEATURE_SIGNING', False)
+    if feature_enabled:
+      self.state = ImplementedTUFMetadataAPI(app, config, client=client)
+    else:
+      self.state = NoopTUFMetadataAPI()
+
+  def __getattr__(self, name):
+    return getattr(self.state, name, None)
+
+
+@add_metaclass(ABCMeta)
+class TUFMetadataAPIInterface(object):
   """ Helper class for talking to the TUF Metadata service (Apostille). """
+
+  @abstractmethod
+  def get_default_tags_with_expiration(self, namespace, repository, targets_file=None):
+    """
+    Gets the tag -> sha mappings for a repo, as well as the expiration of the signatures.
+    Does not verify the metadata, this is purely for display purposes.
+
+    Args:
+      namespace: namespace containing the repository
+      repository: the repo to get tags for
+      targets_file: the specific delegation to read from. Default: targets/releases.json
+
+    Returns:
+      targets, expiration or None, None
+    """
+    pass
+
+  @abstractmethod
+  def delete_metadata(self, namespace, repository):
+    """
+    Deletes the TUF metadata for a repo
+
+    Args:
+      namespace: namespace containing the repository
+      repository: the repo to delete metadata for
+
+    Returns:
+       True if successful, False otherwise
+    """
+    pass
+
+
+@nooper
+class NoopTUFMetadataAPI(TUFMetadataAPIInterface):
+  """ No-op version of the TUF API. """
+  pass
+
+
+class ImplementedTUFMetadataAPI(TUFMetadataAPIInterface):
   def __init__(self, app, config, client=None):
     self._app = app
     self._instance_keys = InstanceKeys(app)
@@ -150,9 +207,6 @@ class TUFMetadataAPI(object):
   def _call(self, method, path, params=None, body=None, headers=None):
     """ Issues an HTTP request to signing service and handles failover for GET requests.
     """
-    if self._config is None:
-      raise Exception('Cannot call unconfigured signing service')
-
     timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1)
     endpoint = self._config['TUF_SERVER']