vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Use the registry API for security scanning

Browse source 
when the storage engine doesn't support direct download url
This commit is contained in:
Jake Moshenko 2016-05-04 17:40:09 -04:00
parent 1ef7008d85
commit 9221a515de
9 changed files with 149 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app.py
View file

@ -195,7 +195,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf
reporter=MetricQueueReporter(metric_queue))
notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf)
secscan_api = SecurityScannerAPI(app.config, storage)
secscan_api = SecurityScannerAPI(app, app.config, storage)
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)

34
auth/registry_jwt_auth.py
View file

@ -16,6 +16,7 @@ from .permissions import repository_read_grant, repository_write_grant
from util.names import parse_namespace_repository
from util.http import abort
from util.security import strictjwt
from util.security.registry_jwt import ANONYMOUS_SUB
from data import model
@ -23,7 +24,6 @@ logger = logging.getLogger(__name__)
TOKEN_REGEX = re.compile(r'^Bearer (([a-zA-Z0-9+/]+\.)+[a-zA-Z0-9+-_/]+)$')
ANONYMOUS_SUB = '(anonymous)'
CONTEXT_KINDS = ['user', 'token', 'oauth']
ACCESS_SCHEMA = {
@ -125,38 +125,6 @@ def get_granted_username():
return granted.user.username
def build_context_and_subject(user, token, oauthtoken):
""" Builds the custom context field for the JWT signed token and returns it,
along with the subject for the JWT signed token. """
if oauthtoken:
context = {
'kind': 'oauth',
'user': user.username,
'oauth': oauthtoken.uuid,
}
return (context, user.username)
if user:
context = {
'kind': 'user',
'user': user.username,
}
return (context, user.username)
if token:
context = {
'kind': 'token',
'token': token.code,
}
return (context, None)
context = {
'kind': 'anonymous',
}
return (context, ANONYMOUS_SUB)
def get_auth_headers(repository=None, scopes=None):
""" Returns a dictionary of headers for auth responses. """
headers = {}

41
endpoints/v2/v2auth.py
View file

@ -1,22 +1,20 @@
import logging
import re
import time
import jwt
from flask import request, jsonify, abort
from cachetools import lru_cache
from app import app, userevents
from data import model
from auth.auth import process_auth
from auth.registry_jwt_auth import build_context_and_subject
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermission,
CreateRepositoryPermission)
from endpoints.v2 import v2_bp
from endpoints.decorators import anon_protect
from util.cache import no_cache
from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX
from endpoints.decorators import anon_protect
from util.security.registry_jwt import generate_jwt_object, build_context_and_subject
logger = logging.getLogger(__name__)
@ -26,18 +24,6 @@ SCOPE_REGEX = re.compile(
r'^repository:(([\.a-zA-Z0-9_\-]+/)?[\.a-zA-Z0-9_\-]+):(((push|pull|\*),)*(push|pull|\*))$'
)
@lru_cache(maxsize=1)
def load_certificate_bytes(certificate_file_path):
with open(certificate_file_path) as cert_file:
return ''.join(cert_file.readlines()[1:-1]).rstrip('\n')
@lru_cache(maxsize=1)
def load_private_key(private_key_file_path):
with open(private_key_file_path) as private_key_file:
return private_key_file.read()
@v2_bp.route('/auth')
@process_auth
@no_cache
@ -156,23 +142,8 @@ def generate_registry_jwt():
# Build the signed JWT.
context, subject = build_context_and_subject(user, token, oauthtoken)
token_data = {
'iss': app.config['JWT_AUTH_TOKEN_ISSUER'],
'aud': audience_param,
'nbf': int(time.time()),
'iat': int(time.time()),
'exp': int(time.time() + TOKEN_VALIDITY_LIFETIME_S),
'sub': subject,
'access': access,
'context': context,
}
certificate = load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH'])
jwt_obj = generate_jwt_object(audience_param, subject, context, access, TOKEN_VALIDITY_LIFETIME_S,
app.config)
token_headers = {
'x5c': [certificate],
}
private_key = load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH'])
return jsonify({'token':jwt.encode(token_data, private_key, 'RS256', headers=token_headers)})
return jsonify({'token': jwt_obj})

12
test/test_registry_v2_auth.py
View file

@ -6,16 +6,18 @@ from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa
from app import app
from endpoints.v2.v2auth import TOKEN_VALIDITY_LIFETIME_S, load_certificate_bytes, load_private_key
from auth.registry_jwt_auth import (identity_from_bearer_token, load_public_key,
InvalidJWTException, build_context_and_subject, ANONYMOUS_SUB)
from endpoints.v2.v2auth import TOKEN_VALIDITY_LIFETIME_S
from auth.registry_jwt_auth import identity_from_bearer_token, load_public_key, InvalidJWTException
from util.morecollections import AttrDict
from util.security.registry_jwt import (_load_certificate_bytes, _load_private_key, ANONYMOUS_SUB,
build_context_and_subject)
TEST_AUDIENCE = app.config['SERVER_HOSTNAME']
TEST_USER = AttrDict({'username': 'joeuser'})
MAX_SIGNED_S = 3660
class TestRegistryV2Auth(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestRegistryV2Auth, self).__init__(*args, **kwargs)
@ -41,13 +43,13 @@ class TestRegistryV2Auth(unittest.TestCase):
def _generate_token(self, token_data):
certificate = load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH'])
certificate = _load_certificate_bytes(app.config['JWT_AUTH_CERTIFICATE_PATH'])
token_headers = {
'x5c': [certificate],
}
private_key = load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH'])
private_key = _load_private_key(app.config['JWT_AUTH_PRIVATE_KEY_PATH'])
token_data = jwt.encode(token_data, private_key, 'RS256', headers=token_headers)
return 'Bearer {0}'.format(token_data)

2
test/test_secscan.py
View file

@ -122,7 +122,7 @@ class TestSecurityScanner(unittest.TestCase):
self.ctx = app.test_request_context()
self.ctx.__enter__()
self.api = SecurityScannerAPI(app.config, storage)
self.api = SecurityScannerAPI(app, app.config, storage)
def tearDown(self):
storage.put_content(['local_us'], 'supports_direct_download', 'false')

2
util/config/validator.py
View file

@ -446,7 +446,7 @@ def _validate_security_scanner(config, _):
# Make a ping request to the security service.
client = app.config['HTTPCLIENT']
api = SecurityScannerAPI(config, None, client=client, skip_validation=True)
api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)
response = api.ping()
if response.status_code != 200:
message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)

83
util/secscan/api.py
View file

@ -1,15 +1,23 @@
import logging
import requests
from flask import url_for
from urlparse import urljoin
from data.database import CloseForLongOperation
from data import model
from data.model.storage import get_storage_locations
from urlparse import urljoin
from util.secscan.validator import SecurityConfigValidator
from util.security.registry_jwt import generate_jwt_object, build_context_and_subject
from util import get_app_url
TOKEN_VALIDITY_LIFETIME_S = 60 # Amount of time the security scanner has to call the layer URL
logger = logging.getLogger(__name__)
class AnalyzeLayerException(Exception):
""" Exception raised when a layer fails to analyze due to a *client-side* issue. """
@ -26,13 +34,14 @@ _API_METHOD_PING = 'metrics'
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (Clair). """
def __init__(self, config, storage, client=None, skip_validation=False):
def __init__(self, app, config, storage, client=None, skip_validation=False):
if not skip_validation:
config_validator = SecurityConfigValidator(config)
if not config_validator.valid():
logger.warning('Invalid config provided to SecurityScannerAPI')
return
self._app = app
self._config = config
self._client = client or config['HTTPCLIENT']
self._storage = storage
@ -40,9 +49,10 @@ class SecurityScannerAPI(object):
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
def _get_image_url(self, image):
""" Gets the download URL for an image and if the storage doesn't exist,
returns None.
def _get_image_url_and_auth(self, image):
""" Returns a tuple of the url and the auth header value that must be used
to fetch the layer data itself. If the image can't be addressed, we return
None.
"""
path = model.storage.get_layer_path(image.storage)
locations = self._default_storage_locations
@ -53,47 +63,60 @@ class SecurityScannerAPI(object):
if not locations or not self._storage.exists(locations, path):
logger.warning('Could not find a valid location to download layer %s.%s out of %s',
image.docker_image_id, image.storage.uuid, locations)
return None
return None, None
uri = self._storage.get_direct_download_url(locations, path)
auth_header = None
if uri is None:
# Handle local storage.
local_storage_enabled = False
for storage_type, _ in self._config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
if storage_type == 'LocalStorage':
local_storage_enabled = True
# Use the registry API instead, with a signed JWT giving access
repo_name = image.repository.name
namespace_name = image.repository.namespace_user.username
repository_and_namespace = '/'.join([namespace_name, repo_name])
if local_storage_enabled:
# TODO: fix to use the proper local storage path.
uri = path
else:
logger.warning('Could not get image URL and local storage was not enabled')
return None
# Generate the JWT which will authorize this
audience = 'security_scanner'
context, subject = build_context_and_subject(None, None, None)
access = [{
'type': 'repository',
'name': repository_and_namespace,
'actions': ['pull'],
}]
auth_jwt = generate_jwt_object(audience, subject, context, access, TOKEN_VALIDITY_LIFETIME_S,
self._config)
auth_header = 'Bearer: {}'.format(auth_jwt)
return uri
with self._app.test_request_context('/'):
relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
digest=image.storage.content_checksum)
uri = urljoin(get_app_url(self._config), relative_layer_url)
return uri, auth_header
def _new_analyze_request(self, image):
""" Create the request body to submit the given image for analysis. If the image's URL cannot
be found, returns None.
"""
url = self._get_image_url(image)
url, auth_header = self._get_image_url_and_auth(image)
if url is None:
return None
request = {
'Layer': {
'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid),
'Path': url,
'Format': 'Docker'
}
layer_request = {
'Name': '%s.%s' % (image.docker_image_id, image.storage.uuid),
'Path': url,
'Format': 'Docker',
}
if image.parent.docker_image_id and image.parent.storage.uuid:
request['Layer']['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
image.parent.storage.uuid)
if auth_header is not None:
layer_request['Authorization'] = auth_header
return request
if image.parent.docker_image_id and image.parent.storage.uuid:
layer_request['ParentName'] = '%s.%s' % (image.parent.docker_image_id,
image.parent.storage.uuid)
return {
'Layer': layer_request,
}
def ping(self):

76
util/security/registry_jwt.py Normal file
View file

@ -0,0 +1,76 @@
import time
import jwt
from cachetools import lru_cache
ANONYMOUS_SUB = '(anonymous)'
def generate_jwt_object(audience, subject, context, access, lifetime_s, app_config):
""" Generates a compact encoded JWT with the values specified.
"""
token_data = {
'iss': app_config['JWT_AUTH_TOKEN_ISSUER'],
'aud': audience,
'nbf': int(time.time()),
'iat': int(time.time()),
'exp': int(time.time() + lifetime_s),
'sub': subject,
'access': access,
'context': context,
}
certificate = _load_certificate_bytes(app_config['JWT_AUTH_CERTIFICATE_PATH'])
token_headers = {
'x5c': [certificate],
}
private_key = _load_private_key(app_config['JWT_AUTH_PRIVATE_KEY_PATH'])
return jwt.encode(token_data, private_key, 'RS256', headers=token_headers)
def build_context_and_subject(user, token, oauthtoken):
""" Builds the custom context field for the JWT signed token and returns it,
along with the subject for the JWT signed token. """
if oauthtoken:
context = {
'kind': 'oauth',
'user': user.username,
'oauth': oauthtoken.uuid,
}
return (context, user.username)
if user:
context = {
'kind': 'user',
'user': user.username,
}
return (context, user.username)
if token:
context = {
'kind': 'token',
'token': token.code,
}
return (context, None)
context = {
'kind': 'anonymous',
}
return (context, ANONYMOUS_SUB)
@lru_cache(maxsize=1)
def _load_certificate_bytes(certificate_file_path):
with open(certificate_file_path) as cert_file:
return ''.join(cert_file.readlines()[1:-1]).rstrip('\n')
@lru_cache(maxsize=1)
def _load_private_key(private_key_file_path):
with open(private_key_file_path) as private_key_file:
return private_key_file.read()

3
workers/securityworker.py
View file

@ -13,6 +13,7 @@ from data.model.image import get_image_with_storage_and_parent_base
from util.secscan.api import SecurityConfigValidator
from util.secscan.analyzer import LayerAnalyzer
from util.migrate.allocator import yield_random_entries
from endpoints.v2 import v2_bp
BATCH_SIZE = 50
INDEXING_INTERVAL = 30
@ -61,6 +62,8 @@ class SecurityWorker(Worker):
self._min_id = max_id + 1
if __name__ == '__main__':
app.register_blueprint(v2_bp, url_prefix='/v2')
if not features.SECURITY_SCANNER:
logger.debug('Security scanner disabled; skipping SecurityWorker')
while True: