Use the registry API for security scanning

when the storage engine doesn't support direct download url

app.py

@@ -195,7 +195,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf
                                    reporter=MetricQueueReporter(metric_queue))
 notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
 secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf)
-secscan_api = SecurityScannerAPI(app.config, storage)
+secscan_api = SecurityScannerAPI(app, app.config, storage)
 
 # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
 _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)

auth/registry_jwt_auth.py

@@ -16,6 +16,7 @@ from .permissions import repository_read_grant, repository_write_grant
 from util.names import parse_namespace_repository
 from util.http import abort
 from util.security import strictjwt
+from util.security.registry_jwt import ANONYMOUS_SUB
 from data import model
 
 
@@ -23,7 +24,6 @@ logger = logging.getLogger(__name__)
 
 
 TOKEN_REGEX = re.compile(r'^Bearer (([a-zA-Z0-9+/]+\.)+[a-zA-Z0-9+-_/]+)$')
-ANONYMOUS_SUB = '(anonymous)'
 CONTEXT_KINDS = ['user', 'token', 'oauth']
 
 ACCESS_SCHEMA = {
@@ -125,38 +125,6 @@ def get_granted_username():
   return granted.user.username
 
 
-def build_context_and_subject(user, token, oauthtoken):
-  """ Builds the custom context field for the JWT signed token and returns it,
-      along with the subject for the JWT signed token. """
-  if oauthtoken:
-    context = {
-      'kind': 'oauth',
-      'user': user.username,
-      'oauth': oauthtoken.uuid,
-    }
-
-    return (context, user.username)
-
-  if user:
-    context = {
-      'kind': 'user',
-      'user': user.username,
-    }
-    return (context, user.username)
-
-  if token:
-    context = {
-      'kind': 'token',
-      'token': token.code,
-    }
-    return (context, None)
-
-  context = {
-    'kind': 'anonymous',
-  }
-  return (context, ANONYMOUS_SUB)
-
-
 def get_auth_headers(repository=None, scopes=None):
   """ Returns a dictionary of headers for auth responses. """
   headers = {}

endpoints/v2/v2auth.py

@@ -1,22 +1,20 @@
 import logging
 import re
-import time
-import jwt
 
 from flask import request, jsonify, abort
-from cachetools import lru_cache
 
 from app import app, userevents
 from data import model
 from auth.auth import process_auth
-from auth.registry_jwt_auth import build_context_and_subject
 from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
 from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermission,
                               CreateRepositoryPermission)
 from endpoints.v2 import v2_bp
+from endpoints.decorators import anon_protect
 from util.cache import no_cache
 from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX
-from endpoints.decorators import anon_protect
+from util.security.registry_jwt import generate_jwt_object, build_context_and_subject
+
 
 logger = logging.getLogger(__name__)
 
@@ -26,18 +24,6 @@ SCOPE_REGEX = re.compile(
     r'^repository:(([\.a-zA-Z0-9_\-]+/)?[\.a-zA-Z0-9_\-]+):(((push|pull|\*),)*(push|pull|\*))$'
 )
 
-@lru_cache(maxsize=1)
-def load_certificate_bytes(certificate_file_path):
-  with open(certificate_file_path) as cert_file:
-    return ''.join(cert_file.readlines()[1:-1]).rstrip('\n')
-
-
-@lru_cache(maxsize=1)
-def load_private_key(private_key_file_path):
-  with open(private_key_file_path) as private_key_file:
-    return private_key_file.read()
-
-
 @v2_bp.route('/auth')
 @process_auth
 @no_cache
@@ -156,23 +142,8 @@ def generate_registry_jwt():
 
   # Build the signed JWT.
   context, subject = build_context_and_subject(user, token, oauthtoken)
-  token_data = {
-    'iss': app.config['JWT_AUTH_TOKEN_ISSUER'],
-    'aud': audience_param,
-    'nbf': int(time.time()),
-    'iat': int(time.time()),
-    'exp': int(time.time() + TOKEN_VALIDITY_LIFETIME_S),
-    'sub': subject,
-    'access': access,
-    'context': context,
-  }
 
-  certificate = load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH'])
+  jwt_obj = generate_jwt_object(audience_param, subject, context, access, TOKEN_VALIDITY_LIFETIME_S,
+                                app.config)
 
-  token_headers = {
+  return jsonify({'token': jwt_obj})
-    'x5c': [certificate],
-  }
-
-  private_key = load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH'])
-
-  return jsonify({'token':jwt.encode(token_data, private_key, 'RS256', headers=token_headers)})

test/test_registry_v2_auth.py

@@ -6,16 +6,18 @@ from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives.asymmetric import rsa
 
 from app import app
-from endpoints.v2.v2auth import TOKEN_VALIDITY_LIFETIME_S, load_certificate_bytes, load_private_key
+from endpoints.v2.v2auth import TOKEN_VALIDITY_LIFETIME_S
-from auth.registry_jwt_auth import (identity_from_bearer_token, load_public_key,
+from auth.registry_jwt_auth import identity_from_bearer_token, load_public_key, InvalidJWTException
-                                    InvalidJWTException, build_context_and_subject, ANONYMOUS_SUB)
 from util.morecollections import AttrDict
+from util.security.registry_jwt import (_load_certificate_bytes, _load_private_key, ANONYMOUS_SUB,
+                                        build_context_and_subject)
 
 
 TEST_AUDIENCE = app.config['SERVER_HOSTNAME']
 TEST_USER = AttrDict({'username': 'joeuser'})
 MAX_SIGNED_S = 3660
 
+
 class TestRegistryV2Auth(unittest.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestRegistryV2Auth, self).__init__(*args, **kwargs)
@@ -41,13 +43,13 @@ class TestRegistryV2Auth(unittest.TestCase):
 
   def _generate_token(self, token_data):
 
-    certificate = load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH'])
+    certificate = _load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH'])
 
     token_headers = {
       'x5c': [certificate],
     }
 
-    private_key = load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH'])
+    private_key = _load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH'])
     token_data = jwt.encode(token_data, private_key, 'RS256', headers=token_headers)
     return 'Bearer {0}'.format(token_data)
 

test/test_secscan.py

@@ -122,7 +122,7 @@ class TestSecurityScanner(unittest.TestCase):
     self.ctx = app.test_request_context()
     self.ctx.__enter__()
 
-    self.api = SecurityScannerAPI(app.config, storage)
+    self.api = SecurityScannerAPI(app, app.config, storage)
 
   def tearDown(self):
     storage.put_content(['local_us'], 'supports_direct_download', 'false')

util/config/validator.py

@@ -446,7 +446,7 @@ def _validate_security_scanner(config, _):
 
   # Make a ping request to the security service.
   client = app.config['HTTPCLIENT']
-  api = SecurityScannerAPI(config, None, client=client, skip_validation=True)
+  api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)
   response = api.ping()
   if response.status_code != 200:
     message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)

util/secscan/api.py

@@ -1,15 +1,23 @@
 import logging
 import requests
 
+from flask import url_for
+from urlparse import urljoin
+
 from data.database import CloseForLongOperation
 from data import model
 from data.model.storage import get_storage_locations
-
-from urlparse import urljoin
 from util.secscan.validator import SecurityConfigValidator
+from util.security.registry_jwt import generate_jwt_object, build_context_and_subject
+from util import get_app_url
+
+
+TOKEN_VALIDITY_LIFETIME_S = 60  # Amount of time the security scanner has to call the layer URL
+
 
 logger = logging.getLogger(__name__)
 
+
 class AnalyzeLayerException(Exception):
   """ Exception raised when a layer fails to analyze due to a *client-side* issue. """
 
@@ -26,13 +34,14 @@ _API_METHOD_PING = 'metrics'
 
 class SecurityScannerAPI(object):
   """ Helper class for talking to the Security Scan service (Clair). """
-  def __init__(self, config, storage, client=None, skip_validation=False):
+  def __init__(self, app, config, storage, client=None, skip_validation=False):
     if not skip_validation:
       config_validator = SecurityConfigValidator(config)
       if not config_validator.valid():
         logger.warning('Invalid config provided to SecurityScannerAPI')
         return
 
+    self._app = app
     self._config = config
     self._client = client or config['HTTPCLIENT']
     self._storage = storage
@@ -40,9 +49,10 @@ class SecurityScannerAPI(object):
     self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
 
 
-  def _get_image_url(self, image):
+  def _get_image_url_and_auth(self, image):
-    """ Gets the download URL for an image and if the storage doesn't exist,
+    """ Returns a tuple of the url and the auth header value that must be used
-        returns None.
+        to fetch the layer data itself. If the image can't be addressed, we return
+        None.
     """
     path = model.storage.get_layer_path(image.storage)
     locations = self._default_storage_locations
@@ -53,47 +63,60 @@ class SecurityScannerAPI(object):
       if not locations or not self._storage.exists(locations, path):
         logger.warning('Could not find a valid location to download layer %s.%s out of %s',
                        image.docker_image_id, image.storage.uuid, locations)
-        return None
+        return None, None
 
     uri = self._storage.get_direct_download_url(locations, path)
+    auth_header = None
     if uri is None:
-      # Handle local storage.
+      # Use the registry API instead, with a signed JWT giving access
-      local_storage_enabled = False
+      repo_name = image.repository.name
-      for storage_type, _ in self._config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
+      namespace_name = image.repository.namespace_user.username
-        if storage_type == 'LocalStorage':
+      repository_and_namespace = '/'.join([namespace_name, repo_name])
-          local_storage_enabled = True
 
-      if local_storage_enabled:
+      # Generate the JWT which will authorize this
-        # TODO: fix to use the proper local storage path.
+      audience = 'security_scanner'
-        uri = path
+      context, subject = build_context_and_subject(None, None, None)
-      else:
+      access = [{
-        logger.warning('Could not get image URL and local storage was not enabled')
+        'type': 'repository',
-        return None
+        'name': repository_and_namespace,
+        'actions': ['pull'],
+      }]
+      auth_jwt = generate_jwt_object(audience, subject, context, access, TOKEN_VALIDITY_LIFETIME_S,
+                                     self._config)
+      auth_header = 'Bearer: {}'.format(auth_jwt)
 
-    return uri
+      with self._app.test_request_context('/'):
+        relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
+                                     digest=image.storage.content_checksum)
+      uri = urljoin(get_app_url(self._config), relative_layer_url)
+
+    return uri, auth_header
 
 
   def _new_analyze_request(self, image):
     """ Create the request body to submit the given image for analysis. If the image's URL cannot
         be found, returns None.
     """
-    url = self._get_image_url(image)
+    url, auth_header = self._get_image_url_and_auth(image)
     if url is None:
       return None
 
-    request = {
+    layer_request = {
-      'Layer': {
       'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid),
       'Path': url,
-        'Format': 'Docker'
+      'Format': 'Docker',
-      }
     }
 
+    if auth_header is not None:
+      layer_request['Authorization'] = auth_header
+
     if image.parent.docker_image_id and image.parent.storage.uuid:
-      request['Layer']['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
+      layer_request['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
                                                image.parent.storage.uuid)
 
-    return request
+    return {
+      'Layer': layer_request,
+    }
 
 
   def ping(self):

util/security/registry_jwt.py

@@ -0,0 +1,76 @@
+import time
+import jwt
+
+from cachetools import lru_cache
+
+
+ANONYMOUS_SUB = '(anonymous)'
+
+
+def generate_jwt_object(audience, subject, context, access, lifetime_s, app_config):
+  """ Generates a compact encoded JWT with the values specified.
+  """
+  token_data = {
+    'iss': app_config['JWT_AUTH_TOKEN_ISSUER'],
+    'aud': audience,
+    'nbf': int(time.time()),
+    'iat': int(time.time()),
+    'exp': int(time.time() + lifetime_s),
+    'sub': subject,
+    'access': access,
+    'context': context,
+  }
+
+  certificate = _load_certificate_bytes(app_config['JWT_AUTH_CERTIFICATE_PATH'])
+
+  token_headers = {
+    'x5c': [certificate],
+  }
+
+  private_key = _load_private_key(app_config['JWT_AUTH_PRIVATE_KEY_PATH'])
+
+  return jwt.encode(token_data, private_key, 'RS256', headers=token_headers)
+
+
+def build_context_and_subject(user, token, oauthtoken):
+  """ Builds the custom context field for the JWT signed token and returns it,
+      along with the subject for the JWT signed token. """
+  if oauthtoken:
+    context = {
+      'kind': 'oauth',
+      'user': user.username,
+      'oauth': oauthtoken.uuid,
+    }
+
+    return (context, user.username)
+
+  if user:
+    context = {
+      'kind': 'user',
+      'user': user.username,
+    }
+    return (context, user.username)
+
+  if token:
+    context = {
+      'kind': 'token',
+      'token': token.code,
+    }
+    return (context, None)
+
+  context = {
+    'kind': 'anonymous',
+  }
+  return (context, ANONYMOUS_SUB)
+
+
+@lru_cache(maxsize=1)
+def _load_certificate_bytes(certificate_file_path):
+  with open(certificate_file_path) as cert_file:
+    return ''.join(cert_file.readlines()[1:-1]).rstrip('\n')
+
+
+@lru_cache(maxsize=1)
+def _load_private_key(private_key_file_path):
+  with open(private_key_file_path) as private_key_file:
+    return private_key_file.read()

workers/securityworker.py

@@ -13,6 +13,7 @@ from data.model.image import get_image_with_storage_and_parent_base
 from util.secscan.api import SecurityConfigValidator
 from util.secscan.analyzer import LayerAnalyzer
 from util.migrate.allocator import yield_random_entries
+from endpoints.v2 import v2_bp
 
 BATCH_SIZE = 50
 INDEXING_INTERVAL = 30
@@ -61,6 +62,8 @@ class SecurityWorker(Worker):
     self._min_id = max_id + 1
 
 if __name__ == '__main__':
+  app.register_blueprint(v2_bp, url_prefix='/v2')
+
   if not features.SECURITY_SCANNER:
     logger.debug('Security scanner disabled; skipping SecurityWorker')
     while True: