Add tufmetadata endpoint

This commit is contained in:
Evan Cordell 2017-04-05 10:03:27 -04:00
parent 1bfca871ec
commit 9515f18fb6
9 changed files with 282 additions and 1 deletions

2
app.py
View file

@ -45,6 +45,7 @@ from util.metrics.metricqueue import MetricQueue
from util.metrics.prometheus import PrometheusPlugin
from util.saas.cloudwatch import start_cloudwatch_sender
from util.secscan.api import SecurityScannerAPI
from util.tufmetadata.api import TUFMetadataAPI
from util.security.instancekeys import InstanceKeys
from util.security.signing import Signer
@ -195,6 +196,7 @@ all_queues = [image_replication_queue, dockerfile_build_queue, notification_queu
secscan_notification_queue, chunk_cleanup_queue]
secscan_api = SecurityScannerAPI(app, app.config, storage)
tuf_metadata_api = TUFMetadataAPI(app, app.config)
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)

34
endpoints/api/signing.py Normal file
View file

@ -0,0 +1,34 @@
""" List and manage repository vulnerabilities and other security information. """
import logging
import features
from app import tuf_metadata_api
from endpoints.api import (require_repo_read, path_param,
RepositoryParamResource, resource, nickname, show_if,
disallow_for_app_repositories)
logger = logging.getLogger(__name__)
def _default_signed_tags_for_repository(namespace, repository):
""" Fetches the tags in the targets/releases delegation, which is the one the docker client will trust. """
tag_data, _ = tuf_metadata_api.get_default_tags(namespace, repository)
return {
'tags': tag_data.keys()
}
@show_if(features.SIGNING)
@resource('/v1/repository/<apirepopath:repository>/signatures')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
class RepositorySignatures(RepositoryParamResource):
""" Operations for managing the signatures in a repository image. """
@require_repo_read
@nickname('getRepoSignatures')
@disallow_for_app_repositories
def get(self, namespace, repository):
""" Fetches the list of signed tags for the repository"""
return _default_signed_tags_for_repository(namespace, repository)

View file

@ -10,6 +10,7 @@ from endpoints.api.repositorynotification import (RepositoryNotification,
RepositoryNotificationList,
TestRepositoryNotification)
from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
from endpoints.api.signing import RepositorySignatures
from endpoints.api.tag import ListRepositoryTags, RepositoryTag, RepositoryTagImages, RestoreTag
from endpoints.api.trigger import (BuildTriggerList, BuildTrigger, BuildTriggerSubdirs,
BuildTriggerActivate, BuildTriggerAnalyze, ActivateBuildTrigger,
@ -48,6 +49,7 @@ FIELD_ARGS = {'trigger_uuid': '1234', 'field_name': 'foobar'}
(TestRepositoryNotification, 'post', NOTIFICATION_ARGS),
(RepositoryImageSecurity, 'get', IMAGE_ARGS),
(RepositoryManifestSecurity, 'get', MANIFEST_ARGS),
(RepositorySignatures, 'get', None),
(ListRepositoryTags, 'get', None),
(RepositoryTag, 'put', TAG_ARGS),
(RepositoryTag, 'delete', TAG_ARGS),

View file

@ -64,7 +64,7 @@ def generate_registry_jwt(auth_result):
user_event_data = {
'action': 'login',
}
tuf_root = 'quay'
tuf_root = QUAY_TUF_ROOT
if len(scope_param) > 0:
match = get_scope_regex().match(scope_param)

View file

@ -57,6 +57,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
SuperUserRepositoryBuildResource, SuperUserRepositoryBuildStatus)
from endpoints.api.globalmessages import GlobalUserMessage, GlobalUserMessages
from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
from endpoints.api.signing import RepositorySignatures
from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
@ -4485,6 +4486,24 @@ class TestRepositoryManifestSecurity(ApiTestCase):
def test_get_devtable(self):
self._run_test('GET', 404, 'devtable', None)
class TestRepositorySignatures(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)
self._set_url(RepositorySignatures, repository='devtable/simple')
def test_get_anonymous(self):
self._run_test('GET', 401, None, None)
def test_get_freshuser(self):
self._run_test('GET', 403, 'freshuser', None)
def test_get_reader(self):
self._run_test('GET', 403, 'reader', None)
def test_get_devtable(self):
self._run_test('GET', 404, 'devtable', None)
class TestRepositoryManifestLabels(ApiTestCase):
def setUp(self):
ApiTestCase.setUp(self)

View file

@ -74,6 +74,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
SuperUserCustomCertificates, SuperUserCustomCertificate)
from endpoints.api.globalmessages import (GlobalUserMessage, GlobalUserMessages,)
from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
from endpoints.api.signing import RepositorySignatures
from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile,
SuperUserCreateInitialSuperUser)
from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
@ -4506,6 +4507,31 @@ class TestRepositoryImageSecurity(ApiTestCase):
expected_code=200)
class TestRepositorySignatures(ApiTestCase):
def test_get_signatures(self):
self.login(ADMIN_ACCESS_USER)
targets = {
'latest': {
'hashes': {
'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
},
'length': 1500
},
'test_tag': {
'hashes': {
'sha256': '1234123'
},
'length': 50
}
}
with patch('app.tuf_metadata_api') as mock_tuf:
mock_tuf.get_default_tags.return_value = targets
signed_tags_response = self.getJsonResponse(RepositorySignatures, params=dict(namespace='ns', repository='repo'))
self.assertEquals(signed_tags_response, {'tags': ['latest', 'test_tag']})
class TestSuperUserCustomCertificates(ApiTestCase):
def test_custom_certificates(self):
self.login(ADMIN_ACCESS_USER)

View file

127
util/tufmetadata/api.py Normal file
View file

@ -0,0 +1,127 @@
import logging
from urlparse import urljoin
import requests
from data.database import CloseForLongOperation
from util.failover import failover, FailoverException
from util.security.instancekeys import InstanceKeys
from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT
DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
MITM_CERT_PATH = '/conf/mitm.cert'
TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour
logger = logging.getLogger(__name__)
class InvalidMetadataException(Exception):
""" Exception raised when the upstream API metadata that doesn't parse correctly. """
pass
class Non200ResponseException(Exception):
""" Exception raised when the upstream API returns a non-200 HTTP status code. """
def __init__(self, response):
super(Non200ResponseException, self).__init__()
self.response = response
class TUFMetadataAPI(object):
""" Helper class for talking to the TUF Metadata service (Apostille). """
def __init__(self, app, config, client=None):
self._app = app
self._instance_keys = InstanceKeys(app)
self._config = config
self._client = client or config['HTTPCLIENT']
def get_default_tags(self, namespace, repository):
""" Gets the tag -> sha mappings in the 'targets/releases' delegation
"""
gun = "%s/%s" % (namespace, repository)
try:
response = self._get(gun, "targets/releases.json")
json_response = response.json()
targets = self._parse_targets(json_response)
except requests.exceptions.Timeout:
logger.exception('Timeout when trying to get metadata for %s', gun)
return None, True
except requests.exceptions.ConnectionError:
logger.exception('Connection error when trying to get metadata for %s', gun)
return None, True
except (requests.exceptions.RequestException, ValueError):
logger.exception('Failed to get metadata for %s', gun)
return None, False
except Non200ResponseException as ex:
return None, ex.response.status_code != 404 and ex.response.status_code != 400
except InvalidMetadataException as ex:
logger.exception('Failed to parse targets from metadata', str(ex))
return None, False
return targets, False
def _parse_targets(self, json_response):
""" Attempts to parse the targets from a metadata response """
signed = json_response.get('signed')
if not signed:
raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response)
targets = signed.get('targets')
if not targets:
raise InvalidMetadataException("Could not find `targets` in metadata: %s" % json_response)
return targets
def _auth_header(self, gun):
""" Generate a registry auth token for apostille"""
access = [{
'type': 'repository',
'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun),
'actions': ['pull'],
}]
context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT)
token = generate_bearer_token("quay", subject, context, access,
TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
return {'Authorization': 'Bearer %s' % token}
def _get(self, gun, metadata_file):
return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun))
def _request(self, method, endpoint, path, body, headers, params, timeout):
""" Issues an HTTP request to the signing endpoint. """
url = urljoin(endpoint, path)
logger.debug('%sing signing URL %s', method.upper(), url)
headers.update(DEFAULT_HTTP_HEADERS)
resp = self._client.request(method, url, json=body, params=params, timeout=timeout,
verify=MITM_CERT_PATH, headers=headers)
if resp.status_code // 100 != 2:
raise Non200ResponseException(resp)
return resp
def _call(self, method, path, params=None, body=None, headers=None):
""" Issues an HTTP request to signing service and handles failover for GET requests.
"""
if self._config is None:
raise Exception('Cannot call unconfigured signing service')
timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1)
endpoint = self._config['TUF_SERVER']
with CloseForLongOperation(self._config):
# If the request isn't a read do not fail over.
if method != 'GET':
return self._request(method, endpoint, path, body, headers, params, timeout)
# The request is read-only and can failover.
all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', [])
return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {})
for endpoint in all_endpoints])
@failover
def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout):
""" This function auto-retries read-only requests until they return a 2xx status code. """
try:
return request_fn('GET', endpoint, path, body, headers, params, timeout)
except (requests.exceptions.RequestException, Non200ResponseException) as ex:
raise FailoverException(ex)

View file

@ -0,0 +1,71 @@
import pytest
import requests
from mock import mock
from flask import Flask
from test import testconfig
from util.tufmetadata import api
valid_response = {
'signed' : {
'type': 'Targets',
'delegations': {
'keys': {},
'roles': {},
},
'expires': '2020-03-30T18:55:26.594764859-04:00',
'targets': {
'latest': {
'hashes': {
'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
},
'length': 1500
}
},
'version': 2
},
'signatures': [
{
'method': 'ecdsa',
'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw=='
}
]
}
@pytest.mark.parametrize('response_code,response_body,expected', [
(200, valid_response, valid_response['signed']['targets']),
(200, {'garbage': 'data'}, None)
])
def test_get_metadata(response_code, response_body, expected):
app = Flask(__name__)
app.config.from_object(testconfig.TestConfig())
client = mock.Mock()
request = mock.Mock(status_code=response_code)
request.json.return_value = response_body
client.request.return_value = request
tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
tags, _ = tuf_api.get_default_tags('quay', 'quay')
assert tags == expected
@pytest.mark.parametrize('connection_error,response_code,exception', [
(True, 200, requests.exceptions.Timeout),
(True, 200, requests.exceptions.ConnectionError),
(False, 200, requests.exceptions.RequestException),
(False, 200, ValueError),
(True, 500, api.Non200ResponseException(mock.Mock(status_code=500))),
(False, 400, api.Non200ResponseException(mock.Mock(status_code=400))),
(False, 404, api.Non200ResponseException(mock.Mock(status_code=404))),
(False, 200, api.InvalidMetadataException)
])
def test_get_metadata_exception(connection_error, response_code, exception):
app = Flask(__name__)
app.config.from_object(testconfig.TestConfig())
request = mock.Mock(status_code=response_code)
client = mock.Mock(request=request)
client.request.side_effect = exception
tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
tags, _ = tuf_api.get_default_tags('quay', 'quay')
assert tags == None