From 9515f18fb6d13a2bde2d53e4a9852e9788fccc60 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Wed, 5 Apr 2017 10:03:27 -0400 Subject: [PATCH] Add tufmetadata endpoint --- app.py | 2 + endpoints/api/signing.py | 34 +++++ endpoints/api/test/test_disallow_for_apps.py | 2 + endpoints/v2/v2auth.py | 2 +- test/test_api_security.py | 19 +++ test/test_api_usage.py | 26 ++++ util/tufmetadata/__init__.py | 0 util/tufmetadata/api.py | 127 +++++++++++++++++++ util/tufmetadata/test/test_tufmetadata.py | 71 +++++++++++ 9 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 endpoints/api/signing.py create mode 100644 util/tufmetadata/__init__.py create mode 100644 util/tufmetadata/api.py create mode 100644 util/tufmetadata/test/test_tufmetadata.py diff --git a/app.py b/app.py index e168f94ba..587432e47 100644 --- a/app.py +++ b/app.py @@ -45,6 +45,7 @@ from util.metrics.metricqueue import MetricQueue from util.metrics.prometheus import PrometheusPlugin from util.saas.cloudwatch import start_cloudwatch_sender from util.secscan.api import SecurityScannerAPI +from util.tufmetadata.api import TUFMetadataAPI from util.security.instancekeys import InstanceKeys from util.security.signing import Signer @@ -195,6 +196,7 @@ all_queues = [image_replication_queue, dockerfile_build_queue, notification_queu secscan_notification_queue, chunk_cleanup_queue] secscan_api = SecurityScannerAPI(app, app.config, storage) +tuf_metadata_api = TUFMetadataAPI(app, app.config) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) diff --git a/endpoints/api/signing.py b/endpoints/api/signing.py new file mode 100644 index 000000000..1c7716c81 --- /dev/null +++ b/endpoints/api/signing.py @@ -0,0 +1,34 @@ +""" List and manage repository vulnerabilities and other security information. """ + +import logging +import features + +from app import tuf_metadata_api +from endpoints.api import (require_repo_read, path_param, + RepositoryParamResource, resource, nickname, show_if, + disallow_for_app_repositories) + +logger = logging.getLogger(__name__) + + +def _default_signed_tags_for_repository(namespace, repository): + """ Fetches the tags in the targets/releases delegation, which is the one the docker client will trust. """ + tag_data, _ = tuf_metadata_api.get_default_tags(namespace, repository) + return { + 'tags': tag_data.keys() + } + + +@show_if(features.SIGNING) +@resource('/v1/repository//signatures') +@path_param('repository', 'The full path of the repository. e.g. namespace/name') +class RepositorySignatures(RepositoryParamResource): + """ Operations for managing the signatures in a repository image. """ + + @require_repo_read + @nickname('getRepoSignatures') + @disallow_for_app_repositories + def get(self, namespace, repository): + """ Fetches the list of signed tags for the repository""" + return _default_signed_tags_for_repository(namespace, repository) + diff --git a/endpoints/api/test/test_disallow_for_apps.py b/endpoints/api/test/test_disallow_for_apps.py index 23f101425..d84ce7743 100644 --- a/endpoints/api/test/test_disallow_for_apps.py +++ b/endpoints/api/test/test_disallow_for_apps.py @@ -10,6 +10,7 @@ from endpoints.api.repositorynotification import (RepositoryNotification, RepositoryNotificationList, TestRepositoryNotification) from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity +from endpoints.api.signing import RepositorySignatures from endpoints.api.tag import ListRepositoryTags, RepositoryTag, RepositoryTagImages, RestoreTag from endpoints.api.trigger import (BuildTriggerList, BuildTrigger, BuildTriggerSubdirs, BuildTriggerActivate, BuildTriggerAnalyze, ActivateBuildTrigger, @@ -48,6 +49,7 @@ FIELD_ARGS = {'trigger_uuid': '1234', 'field_name': 'foobar'} (TestRepositoryNotification, 'post', NOTIFICATION_ARGS), (RepositoryImageSecurity, 'get', IMAGE_ARGS), (RepositoryManifestSecurity, 'get', MANIFEST_ARGS), + (RepositorySignatures, 'get', None), (ListRepositoryTags, 'get', None), (RepositoryTag, 'put', TAG_ARGS), (RepositoryTag, 'delete', TAG_ARGS), diff --git a/endpoints/v2/v2auth.py b/endpoints/v2/v2auth.py index c304300d4..12e7863a8 100644 --- a/endpoints/v2/v2auth.py +++ b/endpoints/v2/v2auth.py @@ -64,7 +64,7 @@ def generate_registry_jwt(auth_result): user_event_data = { 'action': 'login', } - tuf_root = 'quay' + tuf_root = QUAY_TUF_ROOT if len(scope_param) > 0: match = get_scope_regex().match(scope_param) diff --git a/test/test_api_security.py b/test/test_api_security.py index 6cd2fbdd4..96be3ed4b 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -57,6 +57,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserRepositoryBuildResource, SuperUserRepositoryBuildStatus) from endpoints.api.globalmessages import GlobalUserMessage, GlobalUserMessages from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity +from endpoints.api.signing import RepositorySignatures from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel @@ -4484,6 +4485,24 @@ class TestRepositoryManifestSecurity(ApiTestCase): def test_get_devtable(self): self._run_test('GET', 404, 'devtable', None) + +class TestRepositorySignatures(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RepositorySignatures, repository='devtable/simple') + + def test_get_anonymous(self): + self._run_test('GET', 401, None, None) + + def test_get_freshuser(self): + self._run_test('GET', 403, 'freshuser', None) + + def test_get_reader(self): + self._run_test('GET', 403, 'reader', None) + + def test_get_devtable(self): + self._run_test('GET', 404, 'devtable', None) + class TestRepositoryManifestLabels(ApiTestCase): def setUp(self): diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 8066d83e3..8b8a7ae94 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -74,6 +74,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserCustomCertificates, SuperUserCustomCertificate) from endpoints.api.globalmessages import (GlobalUserMessage, GlobalUserMessages,) from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity +from endpoints.api.signing import RepositorySignatures from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile, SuperUserCreateInitialSuperUser) from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel @@ -4506,6 +4507,31 @@ class TestRepositoryImageSecurity(ApiTestCase): expected_code=200) +class TestRepositorySignatures(ApiTestCase): + def test_get_signatures(self): + self.login(ADMIN_ACCESS_USER) + + targets = { + 'latest': { + 'hashes': { + 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' + }, + 'length': 1500 + }, + 'test_tag': { + 'hashes': { + 'sha256': '1234123' + }, + 'length': 50 + } + } + + with patch('app.tuf_metadata_api') as mock_tuf: + mock_tuf.get_default_tags.return_value = targets + signed_tags_response = self.getJsonResponse(RepositorySignatures, params=dict(namespace='ns', repository='repo')) + self.assertEquals(signed_tags_response, {'tags': ['latest', 'test_tag']}) + + class TestSuperUserCustomCertificates(ApiTestCase): def test_custom_certificates(self): self.login(ADMIN_ACCESS_USER) diff --git a/util/tufmetadata/__init__.py b/util/tufmetadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/tufmetadata/api.py b/util/tufmetadata/api.py new file mode 100644 index 000000000..f30a262cd --- /dev/null +++ b/util/tufmetadata/api.py @@ -0,0 +1,127 @@ +import logging +from urlparse import urljoin + +import requests + +from data.database import CloseForLongOperation +from util.failover import failover, FailoverException +from util.security.instancekeys import InstanceKeys +from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT + + +DEFAULT_HTTP_HEADERS = {'Connection': 'close'} +MITM_CERT_PATH = '/conf/mitm.cert' +TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour + +logger = logging.getLogger(__name__) + + +class InvalidMetadataException(Exception): + """ Exception raised when the upstream API metadata that doesn't parse correctly. """ + pass + + +class Non200ResponseException(Exception): + """ Exception raised when the upstream API returns a non-200 HTTP status code. """ + def __init__(self, response): + super(Non200ResponseException, self).__init__() + self.response = response + + +class TUFMetadataAPI(object): + """ Helper class for talking to the TUF Metadata service (Apostille). """ + def __init__(self, app, config, client=None): + self._app = app + self._instance_keys = InstanceKeys(app) + self._config = config + self._client = client or config['HTTPCLIENT'] + + def get_default_tags(self, namespace, repository): + """ Gets the tag -> sha mappings in the 'targets/releases' delegation + """ + gun = "%s/%s" % (namespace, repository) + try: + response = self._get(gun, "targets/releases.json") + json_response = response.json() + targets = self._parse_targets(json_response) + except requests.exceptions.Timeout: + logger.exception('Timeout when trying to get metadata for %s', gun) + return None, True + except requests.exceptions.ConnectionError: + logger.exception('Connection error when trying to get metadata for %s', gun) + return None, True + except (requests.exceptions.RequestException, ValueError): + logger.exception('Failed to get metadata for %s', gun) + return None, False + except Non200ResponseException as ex: + return None, ex.response.status_code != 404 and ex.response.status_code != 400 + except InvalidMetadataException as ex: + logger.exception('Failed to parse targets from metadata', str(ex)) + return None, False + + return targets, False + + def _parse_targets(self, json_response): + """ Attempts to parse the targets from a metadata response """ + signed = json_response.get('signed') + if not signed: + raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response) + targets = signed.get('targets') + if not targets: + raise InvalidMetadataException("Could not find `targets` in metadata: %s" % json_response) + return targets + + def _auth_header(self, gun): + """ Generate a registry auth token for apostille""" + access = [{ + 'type': 'repository', + 'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun), + 'actions': ['pull'], + }] + context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT) + token = generate_bearer_token("quay", subject, context, access, + TOKEN_VALIDITY_LIFETIME_S, self._instance_keys) + return {'Authorization': 'Bearer %s' % token} + + def _get(self, gun, metadata_file): + return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun)) + + def _request(self, method, endpoint, path, body, headers, params, timeout): + """ Issues an HTTP request to the signing endpoint. """ + url = urljoin(endpoint, path) + logger.debug('%sing signing URL %s', method.upper(), url) + + headers.update(DEFAULT_HTTP_HEADERS) + resp = self._client.request(method, url, json=body, params=params, timeout=timeout, + verify=MITM_CERT_PATH, headers=headers) + if resp.status_code // 100 != 2: + raise Non200ResponseException(resp) + return resp + + def _call(self, method, path, params=None, body=None, headers=None): + """ Issues an HTTP request to signing service and handles failover for GET requests. + """ + if self._config is None: + raise Exception('Cannot call unconfigured signing service') + + timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1) + endpoint = self._config['TUF_SERVER'] + + with CloseForLongOperation(self._config): + # If the request isn't a read do not fail over. + if method != 'GET': + return self._request(method, endpoint, path, body, headers, params, timeout) + + # The request is read-only and can failover. + all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', []) + return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {}) + for endpoint in all_endpoints]) + + +@failover +def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout): + """ This function auto-retries read-only requests until they return a 2xx status code. """ + try: + return request_fn('GET', endpoint, path, body, headers, params, timeout) + except (requests.exceptions.RequestException, Non200ResponseException) as ex: + raise FailoverException(ex) diff --git a/util/tufmetadata/test/test_tufmetadata.py b/util/tufmetadata/test/test_tufmetadata.py new file mode 100644 index 000000000..98f36ccb5 --- /dev/null +++ b/util/tufmetadata/test/test_tufmetadata.py @@ -0,0 +1,71 @@ +import pytest +import requests +from mock import mock + +from flask import Flask + +from test import testconfig +from util.tufmetadata import api + +valid_response = { + 'signed' : { + 'type': 'Targets', + 'delegations': { + 'keys': {}, + 'roles': {}, + }, + 'expires': '2020-03-30T18:55:26.594764859-04:00', + 'targets': { + 'latest': { + 'hashes': { + 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' + }, + 'length': 1500 + } + }, + 'version': 2 + }, + 'signatures': [ + { + 'method': 'ecdsa', + 'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw==' + } + ] +} + + +@pytest.mark.parametrize('response_code,response_body,expected', [ + (200, valid_response, valid_response['signed']['targets']), + (200, {'garbage': 'data'}, None) +]) +def test_get_metadata(response_code, response_body, expected): + app = Flask(__name__) + app.config.from_object(testconfig.TestConfig()) + client = mock.Mock() + request = mock.Mock(status_code=response_code) + request.json.return_value = response_body + client.request.return_value = request + tuf_api = api.TUFMetadataAPI(app, app.config, client=client) + tags, _ = tuf_api.get_default_tags('quay', 'quay') + assert tags == expected + +@pytest.mark.parametrize('connection_error,response_code,exception', [ + (True, 200, requests.exceptions.Timeout), + (True, 200, requests.exceptions.ConnectionError), + (False, 200, requests.exceptions.RequestException), + (False, 200, ValueError), + (True, 500, api.Non200ResponseException(mock.Mock(status_code=500))), + (False, 400, api.Non200ResponseException(mock.Mock(status_code=400))), + (False, 404, api.Non200ResponseException(mock.Mock(status_code=404))), + (False, 200, api.InvalidMetadataException) + +]) +def test_get_metadata_exception(connection_error, response_code, exception): + app = Flask(__name__) + app.config.from_object(testconfig.TestConfig()) + request = mock.Mock(status_code=response_code) + client = mock.Mock(request=request) + client.request.side_effect = exception + tuf_api = api.TUFMetadataAPI(app, app.config, client=client) + tags, _ = tuf_api.get_default_tags('quay', 'quay') + assert tags == None \ No newline at end of file