From 9221a515de9ac23d68d4dd0297d7f99e00c96eb3 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Wed, 4 May 2016 17:40:09 -0400 Subject: [PATCH] Use the registry API for security scanning when the storage engine doesn't support direct download url --- app.py | 2 +- auth/registry_jwt_auth.py | 34 +------------- endpoints/v2/v2auth.py | 41 +++-------------- test/test_registry_v2_auth.py | 12 ++--- test/test_secscan.py | 2 +- util/config/validator.py | 2 +- util/secscan/api.py | 83 ++++++++++++++++++++++------------- util/security/registry_jwt.py | 76 ++++++++++++++++++++++++++++++++ workers/securityworker.py | 3 ++ 9 files changed, 149 insertions(+), 106 deletions(-) create mode 100644 util/security/registry_jwt.py diff --git a/app.py b/app.py index b5a8b6bd7..5dd11062f 100644 --- a/app.py +++ b/app.py @@ -195,7 +195,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf reporter=MetricQueueReporter(metric_queue)) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf) -secscan_api = SecurityScannerAPI(app.config, storage) +secscan_api = SecurityScannerAPI(app, app.config, storage) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) diff --git a/auth/registry_jwt_auth.py b/auth/registry_jwt_auth.py index 97616e761..79f240187 100644 --- a/auth/registry_jwt_auth.py +++ b/auth/registry_jwt_auth.py @@ -16,6 +16,7 @@ from .permissions import repository_read_grant, repository_write_grant from util.names import parse_namespace_repository from util.http import abort from util.security import strictjwt +from util.security.registry_jwt import ANONYMOUS_SUB from data import model @@ -23,7 +24,6 @@ logger = logging.getLogger(__name__) TOKEN_REGEX = re.compile(r'^Bearer (([a-zA-Z0-9+/]+\.)+[a-zA-Z0-9+-_/]+)$') -ANONYMOUS_SUB = '(anonymous)' CONTEXT_KINDS = ['user', 'token', 'oauth'] ACCESS_SCHEMA = { @@ -125,38 +125,6 @@ def get_granted_username(): return granted.user.username -def build_context_and_subject(user, token, oauthtoken): - """ Builds the custom context field for the JWT signed token and returns it, - along with the subject for the JWT signed token. """ - if oauthtoken: - context = { - 'kind': 'oauth', - 'user': user.username, - 'oauth': oauthtoken.uuid, - } - - return (context, user.username) - - if user: - context = { - 'kind': 'user', - 'user': user.username, - } - return (context, user.username) - - if token: - context = { - 'kind': 'token', - 'token': token.code, - } - return (context, None) - - context = { - 'kind': 'anonymous', - } - return (context, ANONYMOUS_SUB) - - def get_auth_headers(repository=None, scopes=None): """ Returns a dictionary of headers for auth responses. """ headers = {} diff --git a/endpoints/v2/v2auth.py b/endpoints/v2/v2auth.py index 184cc1ee7..f5de3943e 100644 --- a/endpoints/v2/v2auth.py +++ b/endpoints/v2/v2auth.py @@ -1,22 +1,20 @@ import logging import re -import time -import jwt from flask import request, jsonify, abort -from cachetools import lru_cache from app import app, userevents from data import model from auth.auth import process_auth -from auth.registry_jwt_auth import build_context_and_subject from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermission, CreateRepositoryPermission) from endpoints.v2 import v2_bp +from endpoints.decorators import anon_protect from util.cache import no_cache from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX -from endpoints.decorators import anon_protect +from util.security.registry_jwt import generate_jwt_object, build_context_and_subject + logger = logging.getLogger(__name__) @@ -26,18 +24,6 @@ SCOPE_REGEX = re.compile( r'^repository:(([\.a-zA-Z0-9_\-]+/)?[\.a-zA-Z0-9_\-]+):(((push|pull|\*),)*(push|pull|\*))$' ) -@lru_cache(maxsize=1) -def load_certificate_bytes(certificate_file_path): - with open(certificate_file_path) as cert_file: - return ''.join(cert_file.readlines()[1:-1]).rstrip('\n') - - -@lru_cache(maxsize=1) -def load_private_key(private_key_file_path): - with open(private_key_file_path) as private_key_file: - return private_key_file.read() - - @v2_bp.route('/auth') @process_auth @no_cache @@ -156,23 +142,8 @@ def generate_registry_jwt(): # Build the signed JWT. context, subject = build_context_and_subject(user, token, oauthtoken) - token_data = { - 'iss': app.config['JWT_AUTH_TOKEN_ISSUER'], - 'aud': audience_param, - 'nbf': int(time.time()), - 'iat': int(time.time()), - 'exp': int(time.time() + TOKEN_VALIDITY_LIFETIME_S), - 'sub': subject, - 'access': access, - 'context': context, - } - certificate = load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH']) + jwt_obj = generate_jwt_object(audience_param, subject, context, access, TOKEN_VALIDITY_LIFETIME_S, + app.config) - token_headers = { - 'x5c': [certificate], - } - - private_key = load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH']) - - return jsonify({'token':jwt.encode(token_data, private_key, 'RS256', headers=token_headers)}) + return jsonify({'token': jwt_obj}) diff --git a/test/test_registry_v2_auth.py b/test/test_registry_v2_auth.py index f72de245d..33cb0b2ff 100644 --- a/test/test_registry_v2_auth.py +++ b/test/test_registry_v2_auth.py @@ -6,16 +6,18 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa from app import app -from endpoints.v2.v2auth import TOKEN_VALIDITY_LIFETIME_S, load_certificate_bytes, load_private_key -from auth.registry_jwt_auth import (identity_from_bearer_token, load_public_key, - InvalidJWTException, build_context_and_subject, ANONYMOUS_SUB) +from endpoints.v2.v2auth import TOKEN_VALIDITY_LIFETIME_S +from auth.registry_jwt_auth import identity_from_bearer_token, load_public_key, InvalidJWTException from util.morecollections import AttrDict +from util.security.registry_jwt import (_load_certificate_bytes, _load_private_key, ANONYMOUS_SUB, + build_context_and_subject) TEST_AUDIENCE = app.config['SERVER_HOSTNAME'] TEST_USER = AttrDict({'username': 'joeuser'}) MAX_SIGNED_S = 3660 + class TestRegistryV2Auth(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestRegistryV2Auth, self).__init__(*args, **kwargs) @@ -41,13 +43,13 @@ class TestRegistryV2Auth(unittest.TestCase): def _generate_token(self, token_data): - certificate = load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH']) + certificate = _load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH']) token_headers = { 'x5c': [certificate], } - private_key = load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH']) + private_key = _load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH']) token_data = jwt.encode(token_data, private_key, 'RS256', headers=token_headers) return 'Bearer {0}'.format(token_data) diff --git a/test/test_secscan.py b/test/test_secscan.py index 219e262f7..fc539ebb6 100644 --- a/test/test_secscan.py +++ b/test/test_secscan.py @@ -122,7 +122,7 @@ class TestSecurityScanner(unittest.TestCase): self.ctx = app.test_request_context() self.ctx.__enter__() - self.api = SecurityScannerAPI(app.config, storage) + self.api = SecurityScannerAPI(app, app.config, storage) def tearDown(self): storage.put_content(['local_us'], 'supports_direct_download', 'false') diff --git a/util/config/validator.py b/util/config/validator.py index 1a8356fcd..80c9b3abb 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -446,7 +446,7 @@ def _validate_security_scanner(config, _): # Make a ping request to the security service. client = app.config['HTTPCLIENT'] - api = SecurityScannerAPI(config, None, client=client, skip_validation=True) + api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True) response = api.ping() if response.status_code != 200: message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text) diff --git a/util/secscan/api.py b/util/secscan/api.py index 8c55cb0c4..9c33007ae 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -1,15 +1,23 @@ import logging import requests +from flask import url_for +from urlparse import urljoin + from data.database import CloseForLongOperation from data import model from data.model.storage import get_storage_locations - -from urlparse import urljoin from util.secscan.validator import SecurityConfigValidator +from util.security.registry_jwt import generate_jwt_object, build_context_and_subject +from util import get_app_url + + +TOKEN_VALIDITY_LIFETIME_S = 60 # Amount of time the security scanner has to call the layer URL + logger = logging.getLogger(__name__) + class AnalyzeLayerException(Exception): """ Exception raised when a layer fails to analyze due to a *client-side* issue. """ @@ -26,13 +34,14 @@ _API_METHOD_PING = 'metrics' class SecurityScannerAPI(object): """ Helper class for talking to the Security Scan service (Clair). """ - def __init__(self, config, storage, client=None, skip_validation=False): + def __init__(self, app, config, storage, client=None, skip_validation=False): if not skip_validation: config_validator = SecurityConfigValidator(config) if not config_validator.valid(): logger.warning('Invalid config provided to SecurityScannerAPI') return + self._app = app self._config = config self._client = client or config['HTTPCLIENT'] self._storage = storage @@ -40,9 +49,10 @@ class SecurityScannerAPI(object): self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2) - def _get_image_url(self, image): - """ Gets the download URL for an image and if the storage doesn't exist, - returns None. + def _get_image_url_and_auth(self, image): + """ Returns a tuple of the url and the auth header value that must be used + to fetch the layer data itself. If the image can't be addressed, we return + None. """ path = model.storage.get_layer_path(image.storage) locations = self._default_storage_locations @@ -53,47 +63,60 @@ class SecurityScannerAPI(object): if not locations or not self._storage.exists(locations, path): logger.warning('Could not find a valid location to download layer %s.%s out of %s', image.docker_image_id, image.storage.uuid, locations) - return None + return None, None uri = self._storage.get_direct_download_url(locations, path) + auth_header = None if uri is None: - # Handle local storage. - local_storage_enabled = False - for storage_type, _ in self._config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values(): - if storage_type == 'LocalStorage': - local_storage_enabled = True + # Use the registry API instead, with a signed JWT giving access + repo_name = image.repository.name + namespace_name = image.repository.namespace_user.username + repository_and_namespace = '/'.join([namespace_name, repo_name]) - if local_storage_enabled: - # TODO: fix to use the proper local storage path. - uri = path - else: - logger.warning('Could not get image URL and local storage was not enabled') - return None + # Generate the JWT which will authorize this + audience = 'security_scanner' + context, subject = build_context_and_subject(None, None, None) + access = [{ + 'type': 'repository', + 'name': repository_and_namespace, + 'actions': ['pull'], + }] + auth_jwt = generate_jwt_object(audience, subject, context, access, TOKEN_VALIDITY_LIFETIME_S, + self._config) + auth_header = 'Bearer: {}'.format(auth_jwt) - return uri + with self._app.test_request_context('/'): + relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace, + digest=image.storage.content_checksum) + uri = urljoin(get_app_url(self._config), relative_layer_url) + + return uri, auth_header def _new_analyze_request(self, image): """ Create the request body to submit the given image for analysis. If the image's URL cannot be found, returns None. """ - url = self._get_image_url(image) + url, auth_header = self._get_image_url_and_auth(image) if url is None: return None - request = { - 'Layer': { - 'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid), - 'Path': url, - 'Format': 'Docker' - } + layer_request = { + 'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid), + 'Path': url, + 'Format': 'Docker', } - if image.parent.docker_image_id and image.parent.storage.uuid: - request['Layer']['ParentName'] = '%s.%s' % (image.parent.docker_image_id, - image.parent.storage.uuid) + if auth_header is not None: + layer_request['Authorization'] = auth_header - return request + if image.parent.docker_image_id and image.parent.storage.uuid: + layer_request['ParentName'] = '%s.%s' % (image.parent.docker_image_id, + image.parent.storage.uuid) + + return { + 'Layer': layer_request, + } def ping(self): diff --git a/util/security/registry_jwt.py b/util/security/registry_jwt.py new file mode 100644 index 000000000..c12e49b43 --- /dev/null +++ b/util/security/registry_jwt.py @@ -0,0 +1,76 @@ +import time +import jwt + +from cachetools import lru_cache + + +ANONYMOUS_SUB = '(anonymous)' + + +def generate_jwt_object(audience, subject, context, access, lifetime_s, app_config): + """ Generates a compact encoded JWT with the values specified. + """ + token_data = { + 'iss': app_config['JWT_AUTH_TOKEN_ISSUER'], + 'aud': audience, + 'nbf': int(time.time()), + 'iat': int(time.time()), + 'exp': int(time.time() + lifetime_s), + 'sub': subject, + 'access': access, + 'context': context, + } + + certificate = _load_certificate_bytes(app_config['JWT_AUTH_CERTIFICATE_PATH']) + + token_headers = { + 'x5c': [certificate], + } + + private_key = _load_private_key(app_config['JWT_AUTH_PRIVATE_KEY_PATH']) + + return jwt.encode(token_data, private_key, 'RS256', headers=token_headers) + + +def build_context_and_subject(user, token, oauthtoken): + """ Builds the custom context field for the JWT signed token and returns it, + along with the subject for the JWT signed token. """ + if oauthtoken: + context = { + 'kind': 'oauth', + 'user': user.username, + 'oauth': oauthtoken.uuid, + } + + return (context, user.username) + + if user: + context = { + 'kind': 'user', + 'user': user.username, + } + return (context, user.username) + + if token: + context = { + 'kind': 'token', + 'token': token.code, + } + return (context, None) + + context = { + 'kind': 'anonymous', + } + return (context, ANONYMOUS_SUB) + + +@lru_cache(maxsize=1) +def _load_certificate_bytes(certificate_file_path): + with open(certificate_file_path) as cert_file: + return ''.join(cert_file.readlines()[1:-1]).rstrip('\n') + + +@lru_cache(maxsize=1) +def _load_private_key(private_key_file_path): + with open(private_key_file_path) as private_key_file: + return private_key_file.read() diff --git a/workers/securityworker.py b/workers/securityworker.py index 7c86962c5..73fb7a0c3 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -13,6 +13,7 @@ from data.model.image import get_image_with_storage_and_parent_base from util.secscan.api import SecurityConfigValidator from util.secscan.analyzer import LayerAnalyzer from util.migrate.allocator import yield_random_entries +from endpoints.v2 import v2_bp BATCH_SIZE = 50 INDEXING_INTERVAL = 30 @@ -61,6 +62,8 @@ class SecurityWorker(Worker): self._min_id = max_id + 1 if __name__ == '__main__': + app.register_blueprint(v2_bp, url_prefix='/v2') + if not features.SECURITY_SCANNER: logger.debug('Security scanner disabled; skipping SecurityWorker') while True: