From 9515f18fb6d13a2bde2d53e4a9852e9788fccc60 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Wed, 5 Apr 2017 10:03:27 -0400 Subject: [PATCH 1/4] Add tufmetadata endpoint --- app.py | 2 + endpoints/api/signing.py | 34 +++++ endpoints/api/test/test_disallow_for_apps.py | 2 + endpoints/v2/v2auth.py | 2 +- test/test_api_security.py | 19 +++ test/test_api_usage.py | 26 ++++ util/tufmetadata/__init__.py | 0 util/tufmetadata/api.py | 127 +++++++++++++++++++ util/tufmetadata/test/test_tufmetadata.py | 71 +++++++++++ 9 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 endpoints/api/signing.py create mode 100644 util/tufmetadata/__init__.py create mode 100644 util/tufmetadata/api.py create mode 100644 util/tufmetadata/test/test_tufmetadata.py diff --git a/app.py b/app.py index e168f94ba..587432e47 100644 --- a/app.py +++ b/app.py @@ -45,6 +45,7 @@ from util.metrics.metricqueue import MetricQueue from util.metrics.prometheus import PrometheusPlugin from util.saas.cloudwatch import start_cloudwatch_sender from util.secscan.api import SecurityScannerAPI +from util.tufmetadata.api import TUFMetadataAPI from util.security.instancekeys import InstanceKeys from util.security.signing import Signer @@ -195,6 +196,7 @@ all_queues = [image_replication_queue, dockerfile_build_queue, notification_queu secscan_notification_queue, chunk_cleanup_queue] secscan_api = SecurityScannerAPI(app, app.config, storage) +tuf_metadata_api = TUFMetadataAPI(app, app.config) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) diff --git a/endpoints/api/signing.py b/endpoints/api/signing.py new file mode 100644 index 000000000..1c7716c81 --- /dev/null +++ b/endpoints/api/signing.py @@ -0,0 +1,34 @@ +""" List and manage repository vulnerabilities and other security information. """ + +import logging +import features + +from app import tuf_metadata_api +from endpoints.api import (require_repo_read, path_param, + RepositoryParamResource, resource, nickname, show_if, + disallow_for_app_repositories) + +logger = logging.getLogger(__name__) + + +def _default_signed_tags_for_repository(namespace, repository): + """ Fetches the tags in the targets/releases delegation, which is the one the docker client will trust. """ + tag_data, _ = tuf_metadata_api.get_default_tags(namespace, repository) + return { + 'tags': tag_data.keys() + } + + +@show_if(features.SIGNING) +@resource('/v1/repository//signatures') +@path_param('repository', 'The full path of the repository. e.g. namespace/name') +class RepositorySignatures(RepositoryParamResource): + """ Operations for managing the signatures in a repository image. """ + + @require_repo_read + @nickname('getRepoSignatures') + @disallow_for_app_repositories + def get(self, namespace, repository): + """ Fetches the list of signed tags for the repository""" + return _default_signed_tags_for_repository(namespace, repository) + diff --git a/endpoints/api/test/test_disallow_for_apps.py b/endpoints/api/test/test_disallow_for_apps.py index 23f101425..d84ce7743 100644 --- a/endpoints/api/test/test_disallow_for_apps.py +++ b/endpoints/api/test/test_disallow_for_apps.py @@ -10,6 +10,7 @@ from endpoints.api.repositorynotification import (RepositoryNotification, RepositoryNotificationList, TestRepositoryNotification) from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity +from endpoints.api.signing import RepositorySignatures from endpoints.api.tag import ListRepositoryTags, RepositoryTag, RepositoryTagImages, RestoreTag from endpoints.api.trigger import (BuildTriggerList, BuildTrigger, BuildTriggerSubdirs, BuildTriggerActivate, BuildTriggerAnalyze, ActivateBuildTrigger, @@ -48,6 +49,7 @@ FIELD_ARGS = {'trigger_uuid': '1234', 'field_name': 'foobar'} (TestRepositoryNotification, 'post', NOTIFICATION_ARGS), (RepositoryImageSecurity, 'get', IMAGE_ARGS), (RepositoryManifestSecurity, 'get', MANIFEST_ARGS), + (RepositorySignatures, 'get', None), (ListRepositoryTags, 'get', None), (RepositoryTag, 'put', TAG_ARGS), (RepositoryTag, 'delete', TAG_ARGS), diff --git a/endpoints/v2/v2auth.py b/endpoints/v2/v2auth.py index c304300d4..12e7863a8 100644 --- a/endpoints/v2/v2auth.py +++ b/endpoints/v2/v2auth.py @@ -64,7 +64,7 @@ def generate_registry_jwt(auth_result): user_event_data = { 'action': 'login', } - tuf_root = 'quay' + tuf_root = QUAY_TUF_ROOT if len(scope_param) > 0: match = get_scope_regex().match(scope_param) diff --git a/test/test_api_security.py b/test/test_api_security.py index 6cd2fbdd4..96be3ed4b 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -57,6 +57,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserRepositoryBuildResource, SuperUserRepositoryBuildStatus) from endpoints.api.globalmessages import GlobalUserMessage, GlobalUserMessages from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity +from endpoints.api.signing import RepositorySignatures from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel @@ -4484,6 +4485,24 @@ class TestRepositoryManifestSecurity(ApiTestCase): def test_get_devtable(self): self._run_test('GET', 404, 'devtable', None) + +class TestRepositorySignatures(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(RepositorySignatures, repository='devtable/simple') + + def test_get_anonymous(self): + self._run_test('GET', 401, None, None) + + def test_get_freshuser(self): + self._run_test('GET', 403, 'freshuser', None) + + def test_get_reader(self): + self._run_test('GET', 403, 'reader', None) + + def test_get_devtable(self): + self._run_test('GET', 404, 'devtable', None) + class TestRepositoryManifestLabels(ApiTestCase): def setUp(self): diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 8066d83e3..8b8a7ae94 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -74,6 +74,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserCustomCertificates, SuperUserCustomCertificate) from endpoints.api.globalmessages import (GlobalUserMessage, GlobalUserMessages,) from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity +from endpoints.api.signing import RepositorySignatures from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile, SuperUserCreateInitialSuperUser) from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel @@ -4506,6 +4507,31 @@ class TestRepositoryImageSecurity(ApiTestCase): expected_code=200) +class TestRepositorySignatures(ApiTestCase): + def test_get_signatures(self): + self.login(ADMIN_ACCESS_USER) + + targets = { + 'latest': { + 'hashes': { + 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' + }, + 'length': 1500 + }, + 'test_tag': { + 'hashes': { + 'sha256': '1234123' + }, + 'length': 50 + } + } + + with patch('app.tuf_metadata_api') as mock_tuf: + mock_tuf.get_default_tags.return_value = targets + signed_tags_response = self.getJsonResponse(RepositorySignatures, params=dict(namespace='ns', repository='repo')) + self.assertEquals(signed_tags_response, {'tags': ['latest', 'test_tag']}) + + class TestSuperUserCustomCertificates(ApiTestCase): def test_custom_certificates(self): self.login(ADMIN_ACCESS_USER) diff --git a/util/tufmetadata/__init__.py b/util/tufmetadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/tufmetadata/api.py b/util/tufmetadata/api.py new file mode 100644 index 000000000..f30a262cd --- /dev/null +++ b/util/tufmetadata/api.py @@ -0,0 +1,127 @@ +import logging +from urlparse import urljoin + +import requests + +from data.database import CloseForLongOperation +from util.failover import failover, FailoverException +from util.security.instancekeys import InstanceKeys +from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT + + +DEFAULT_HTTP_HEADERS = {'Connection': 'close'} +MITM_CERT_PATH = '/conf/mitm.cert' +TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour + +logger = logging.getLogger(__name__) + + +class InvalidMetadataException(Exception): + """ Exception raised when the upstream API metadata that doesn't parse correctly. """ + pass + + +class Non200ResponseException(Exception): + """ Exception raised when the upstream API returns a non-200 HTTP status code. """ + def __init__(self, response): + super(Non200ResponseException, self).__init__() + self.response = response + + +class TUFMetadataAPI(object): + """ Helper class for talking to the TUF Metadata service (Apostille). """ + def __init__(self, app, config, client=None): + self._app = app + self._instance_keys = InstanceKeys(app) + self._config = config + self._client = client or config['HTTPCLIENT'] + + def get_default_tags(self, namespace, repository): + """ Gets the tag -> sha mappings in the 'targets/releases' delegation + """ + gun = "%s/%s" % (namespace, repository) + try: + response = self._get(gun, "targets/releases.json") + json_response = response.json() + targets = self._parse_targets(json_response) + except requests.exceptions.Timeout: + logger.exception('Timeout when trying to get metadata for %s', gun) + return None, True + except requests.exceptions.ConnectionError: + logger.exception('Connection error when trying to get metadata for %s', gun) + return None, True + except (requests.exceptions.RequestException, ValueError): + logger.exception('Failed to get metadata for %s', gun) + return None, False + except Non200ResponseException as ex: + return None, ex.response.status_code != 404 and ex.response.status_code != 400 + except InvalidMetadataException as ex: + logger.exception('Failed to parse targets from metadata', str(ex)) + return None, False + + return targets, False + + def _parse_targets(self, json_response): + """ Attempts to parse the targets from a metadata response """ + signed = json_response.get('signed') + if not signed: + raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response) + targets = signed.get('targets') + if not targets: + raise InvalidMetadataException("Could not find `targets` in metadata: %s" % json_response) + return targets + + def _auth_header(self, gun): + """ Generate a registry auth token for apostille""" + access = [{ + 'type': 'repository', + 'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun), + 'actions': ['pull'], + }] + context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT) + token = generate_bearer_token("quay", subject, context, access, + TOKEN_VALIDITY_LIFETIME_S, self._instance_keys) + return {'Authorization': 'Bearer %s' % token} + + def _get(self, gun, metadata_file): + return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun)) + + def _request(self, method, endpoint, path, body, headers, params, timeout): + """ Issues an HTTP request to the signing endpoint. """ + url = urljoin(endpoint, path) + logger.debug('%sing signing URL %s', method.upper(), url) + + headers.update(DEFAULT_HTTP_HEADERS) + resp = self._client.request(method, url, json=body, params=params, timeout=timeout, + verify=MITM_CERT_PATH, headers=headers) + if resp.status_code // 100 != 2: + raise Non200ResponseException(resp) + return resp + + def _call(self, method, path, params=None, body=None, headers=None): + """ Issues an HTTP request to signing service and handles failover for GET requests. + """ + if self._config is None: + raise Exception('Cannot call unconfigured signing service') + + timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1) + endpoint = self._config['TUF_SERVER'] + + with CloseForLongOperation(self._config): + # If the request isn't a read do not fail over. + if method != 'GET': + return self._request(method, endpoint, path, body, headers, params, timeout) + + # The request is read-only and can failover. + all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', []) + return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {}) + for endpoint in all_endpoints]) + + +@failover +def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout): + """ This function auto-retries read-only requests until they return a 2xx status code. """ + try: + return request_fn('GET', endpoint, path, body, headers, params, timeout) + except (requests.exceptions.RequestException, Non200ResponseException) as ex: + raise FailoverException(ex) diff --git a/util/tufmetadata/test/test_tufmetadata.py b/util/tufmetadata/test/test_tufmetadata.py new file mode 100644 index 000000000..98f36ccb5 --- /dev/null +++ b/util/tufmetadata/test/test_tufmetadata.py @@ -0,0 +1,71 @@ +import pytest +import requests +from mock import mock + +from flask import Flask + +from test import testconfig +from util.tufmetadata import api + +valid_response = { + 'signed' : { + 'type': 'Targets', + 'delegations': { + 'keys': {}, + 'roles': {}, + }, + 'expires': '2020-03-30T18:55:26.594764859-04:00', + 'targets': { + 'latest': { + 'hashes': { + 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' + }, + 'length': 1500 + } + }, + 'version': 2 + }, + 'signatures': [ + { + 'method': 'ecdsa', + 'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw==' + } + ] +} + + +@pytest.mark.parametrize('response_code,response_body,expected', [ + (200, valid_response, valid_response['signed']['targets']), + (200, {'garbage': 'data'}, None) +]) +def test_get_metadata(response_code, response_body, expected): + app = Flask(__name__) + app.config.from_object(testconfig.TestConfig()) + client = mock.Mock() + request = mock.Mock(status_code=response_code) + request.json.return_value = response_body + client.request.return_value = request + tuf_api = api.TUFMetadataAPI(app, app.config, client=client) + tags, _ = tuf_api.get_default_tags('quay', 'quay') + assert tags == expected + +@pytest.mark.parametrize('connection_error,response_code,exception', [ + (True, 200, requests.exceptions.Timeout), + (True, 200, requests.exceptions.ConnectionError), + (False, 200, requests.exceptions.RequestException), + (False, 200, ValueError), + (True, 500, api.Non200ResponseException(mock.Mock(status_code=500))), + (False, 400, api.Non200ResponseException(mock.Mock(status_code=400))), + (False, 404, api.Non200ResponseException(mock.Mock(status_code=404))), + (False, 200, api.InvalidMetadataException) + +]) +def test_get_metadata_exception(connection_error, response_code, exception): + app = Flask(__name__) + app.config.from_object(testconfig.TestConfig()) + request = mock.Mock(status_code=response_code) + client = mock.Mock(request=request) + client.request.side_effect = exception + tuf_api = api.TUFMetadataAPI(app, app.config, client=client) + tags, _ = tuf_api.get_default_tags('quay', 'quay') + assert tags == None \ No newline at end of file From 1a787225216cfab7e4377db212c87e4f3d9d9702 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Wed, 5 Apr 2017 13:27:31 -0400 Subject: [PATCH 2/4] Update tests --- endpoints/api/signing.py | 3 ++ endpoints/api/test/test_security.py | 6 ++++ endpoints/api/test/test_signing.py | 43 +++++++++++++++++++++++++++++ test/test_api_security.py | 18 ------------ test/test_api_usage.py | 24 ---------------- test/testconfig.py | 2 ++ 6 files changed, 54 insertions(+), 42 deletions(-) create mode 100644 endpoints/api/test/test_signing.py diff --git a/endpoints/api/signing.py b/endpoints/api/signing.py index 1c7716c81..34096187b 100644 --- a/endpoints/api/signing.py +++ b/endpoints/api/signing.py @@ -14,6 +14,9 @@ logger = logging.getLogger(__name__) def _default_signed_tags_for_repository(namespace, repository): """ Fetches the tags in the targets/releases delegation, which is the one the docker client will trust. """ tag_data, _ = tuf_metadata_api.get_default_tags(namespace, repository) + if not tag_data: + return {'tags': None} + return { 'tags': tag_data.keys() } diff --git a/endpoints/api/test/test_security.py b/endpoints/api/test/test_security.py index 1ae2c31cc..c9b72c87c 100644 --- a/endpoints/api/test/test_security.py +++ b/endpoints/api/test/test_security.py @@ -5,6 +5,7 @@ from endpoints.api.team import OrganizationTeamSyncing from endpoints.api.test.shared import client_with_identity, conduct_api_call from endpoints.api.superuser import SuperUserRepositoryBuildLogs, SuperUserRepositoryBuildResource from endpoints.api.superuser import SuperUserRepositoryBuildStatus +from endpoints.api.signing import RepositorySignatures from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file TEAM_PARAMS = {'orgname': 'buynlarge', 'teamname': 'owners'} @@ -35,6 +36,11 @@ BUILD_PARAMS = {'build_uuid': 'test-1234'} (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'freshuser', 403), (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'reader', 403), (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'devtable', 404), + + (RepositorySignatures, 'GET', 401, None, None), + (RepositorySignatures, 'GET', 403, 'freshuser', None), + (RepositorySignatures, 'GET', 403, 'reader', None), + (RepositorySignatures, 'GET', 404, 'devtable', None), ]) def test_api_security(resource, method, params, body, identity, expected, client): with client_with_identity(identity, client) as cl: diff --git a/endpoints/api/test/test_signing.py b/endpoints/api/test/test_signing.py new file mode 100644 index 000000000..5d7744be5 --- /dev/null +++ b/endpoints/api/test/test_signing.py @@ -0,0 +1,43 @@ +from collections import Counter + +import pytest + +from endpoints.api.test.shared import client_with_identity, conduct_api_call +from endpoints.api.signing import RepositorySignatures +from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file +from mock import patch + +VALID_TARGETS = { + 'latest': { + 'hashes': { + 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' + }, + 'length': 1500 + }, + 'test_tag': { + 'hashes': { + 'sha256': '1234123' + }, + 'length': 50 + } +} + +def tags_equal(expected, actual): + expected_tags = expected.get('tags') + actual_tags = actual.get('tags') + if expected_tags and actual_tags: + return Counter(expected_tags) == Counter(actual_tags) + return expected == actual + +@pytest.mark.parametrize('targets,expected', [ + (VALID_TARGETS, {'tags':['latest', 'test_tag']}), + ({'bad': 'tags'}, ({'tags': ['bad']})), + ({}, ({'tags': None})), + (None, ({'tags': None})), # API returns None on exceptions +]) +def test_get_signatures(targets, expected, client): + with patch('endpoints.api.signing.tuf_metadata_api') as mock_tuf: + mock_tuf.get_default_tags.return_value = (targets, False) + with client_with_identity('devtable', client) as cl: + params = {'repository': 'devtable/repo'} + assert tags_equal(expected, conduct_api_call(cl, RepositorySignatures, 'GET', params, None, 200).json) diff --git a/test/test_api_security.py b/test/test_api_security.py index 96be3ed4b..0df692b17 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -57,7 +57,6 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserRepositoryBuildResource, SuperUserRepositoryBuildStatus) from endpoints.api.globalmessages import GlobalUserMessage, GlobalUserMessages from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity -from endpoints.api.signing import RepositorySignatures from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel @@ -4485,23 +4484,6 @@ class TestRepositoryManifestSecurity(ApiTestCase): def test_get_devtable(self): self._run_test('GET', 404, 'devtable', None) - -class TestRepositorySignatures(ApiTestCase): - def setUp(self): - ApiTestCase.setUp(self) - self._set_url(RepositorySignatures, repository='devtable/simple') - - def test_get_anonymous(self): - self._run_test('GET', 401, None, None) - - def test_get_freshuser(self): - self._run_test('GET', 403, 'freshuser', None) - - def test_get_reader(self): - self._run_test('GET', 403, 'reader', None) - - def test_get_devtable(self): - self._run_test('GET', 404, 'devtable', None) class TestRepositoryManifestLabels(ApiTestCase): diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 8b8a7ae94..89495262e 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -74,7 +74,6 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserCustomCertificates, SuperUserCustomCertificate) from endpoints.api.globalmessages import (GlobalUserMessage, GlobalUserMessages,) from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity -from endpoints.api.signing import RepositorySignatures from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile, SuperUserCreateInitialSuperUser) from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel @@ -4507,29 +4506,6 @@ class TestRepositoryImageSecurity(ApiTestCase): expected_code=200) -class TestRepositorySignatures(ApiTestCase): - def test_get_signatures(self): - self.login(ADMIN_ACCESS_USER) - - targets = { - 'latest': { - 'hashes': { - 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' - }, - 'length': 1500 - }, - 'test_tag': { - 'hashes': { - 'sha256': '1234123' - }, - 'length': 50 - } - } - - with patch('app.tuf_metadata_api') as mock_tuf: - mock_tuf.get_default_tags.return_value = targets - signed_tags_response = self.getJsonResponse(RepositorySignatures, params=dict(namespace='ns', repository='repo')) - self.assertEquals(signed_tags_response, {'tags': ['latest', 'test_tag']}) class TestSuperUserCustomCertificates(ApiTestCase): diff --git a/test/testconfig.py b/test/testconfig.py index 72c5fb229..ab2a238b9 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -64,6 +64,8 @@ class TestConfig(DefaultConfig): SECURITY_SCANNER_API_VERSION = 'v1' SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1 SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1 + + FEATURE_SIGNING = True SIGNING_ENGINE = 'gpg2' From 217b4a5ab25b9049da0b0be5ca92e1dbe6242042 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Fri, 7 Apr 2017 16:12:28 -0400 Subject: [PATCH 3/4] Return hashes and expiration when fetching signed tags --- endpoints/api/signing.py | 19 ++++-------- endpoints/api/test/test_security.py | 9 +++--- endpoints/api/test/test_signing.py | 10 +++---- util/tufmetadata/api.py | 35 ++++++++++++----------- util/tufmetadata/test/test_tufmetadata.py | 13 +++++---- 5 files changed, 42 insertions(+), 44 deletions(-) diff --git a/endpoints/api/signing.py b/endpoints/api/signing.py index 34096187b..454482850 100644 --- a/endpoints/api/signing.py +++ b/endpoints/api/signing.py @@ -11,17 +11,6 @@ from endpoints.api import (require_repo_read, path_param, logger = logging.getLogger(__name__) -def _default_signed_tags_for_repository(namespace, repository): - """ Fetches the tags in the targets/releases delegation, which is the one the docker client will trust. """ - tag_data, _ = tuf_metadata_api.get_default_tags(namespace, repository) - if not tag_data: - return {'tags': None} - - return { - 'tags': tag_data.keys() - } - - @show_if(features.SIGNING) @resource('/v1/repository//signatures') @path_param('repository', 'The full path of the repository. e.g. namespace/name') @@ -32,6 +21,10 @@ class RepositorySignatures(RepositoryParamResource): @nickname('getRepoSignatures') @disallow_for_app_repositories def get(self, namespace, repository): - """ Fetches the list of signed tags for the repository""" - return _default_signed_tags_for_repository(namespace, repository) + """ Fetches the list of signed tags for the repository""" + tag_data, expiration = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository) + return { + 'tags': tag_data, + 'expiration': expiration + } diff --git a/endpoints/api/test/test_security.py b/endpoints/api/test/test_security.py index c9b72c87c..425ad1682 100644 --- a/endpoints/api/test/test_security.py +++ b/endpoints/api/test/test_security.py @@ -1,4 +1,5 @@ import pytest +from flask_principal import AnonymousIdentity from endpoints.api import api from endpoints.api.team import OrganizationTeamSyncing @@ -10,6 +11,7 @@ from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_f TEAM_PARAMS = {'orgname': 'buynlarge', 'teamname': 'owners'} BUILD_PARAMS = {'build_uuid': 'test-1234'} +REPO_PARAMS = {'repository': 'devtable/someapp'} @pytest.mark.parametrize('resource,method,params,body,identity,expected', [ (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, None, 403), @@ -37,10 +39,9 @@ BUILD_PARAMS = {'build_uuid': 'test-1234'} (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'reader', 403), (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'devtable', 404), - (RepositorySignatures, 'GET', 401, None, None), - (RepositorySignatures, 'GET', 403, 'freshuser', None), - (RepositorySignatures, 'GET', 403, 'reader', None), - (RepositorySignatures, 'GET', 404, 'devtable', None), + (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'freshuser', 403), + (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'reader', 403), + (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'devtable', 200), ]) def test_api_security(resource, method, params, body, identity, expected, client): with client_with_identity(identity, client) as cl: diff --git a/endpoints/api/test/test_signing.py b/endpoints/api/test/test_signing.py index 5d7744be5..93ac94be6 100644 --- a/endpoints/api/test/test_signing.py +++ b/endpoints/api/test/test_signing.py @@ -30,14 +30,14 @@ def tags_equal(expected, actual): return expected == actual @pytest.mark.parametrize('targets,expected', [ - (VALID_TARGETS, {'tags':['latest', 'test_tag']}), - ({'bad': 'tags'}, ({'tags': ['bad']})), - ({}, ({'tags': None})), - (None, ({'tags': None})), # API returns None on exceptions + (VALID_TARGETS, {'tags': VALID_TARGETS, 'expiration': 'expires'}), + ({'bad': 'tags'}, {'tags': {'bad': 'tags'}, 'expiration': 'expires'}), + ({}, {'tags': {}, 'expiration': 'expires'}), + (None, {'tags': None, 'expiration': 'expires'}), # API returns None on exceptions ]) def test_get_signatures(targets, expected, client): with patch('endpoints.api.signing.tuf_metadata_api') as mock_tuf: - mock_tuf.get_default_tags.return_value = (targets, False) + mock_tuf.get_default_tags_with_expiration.return_value = (targets, 'expires') with client_with_identity('devtable', client) as cl: params = {'repository': 'devtable/repo'} assert tags_equal(expected, conduct_api_call(cl, RepositorySignatures, 'GET', params, None, 200).json) diff --git a/util/tufmetadata/api.py b/util/tufmetadata/api.py index f30a262cd..2ece0c307 100644 --- a/util/tufmetadata/api.py +++ b/util/tufmetadata/api.py @@ -36,40 +36,43 @@ class TUFMetadataAPI(object): self._config = config self._client = client or config['HTTPCLIENT'] - def get_default_tags(self, namespace, repository): - """ Gets the tag -> sha mappings in the 'targets/releases' delegation + def get_default_tags_with_expiration(self, namespace, repository, targets_file=None): + """ Gets the tag -> sha mappings in the 'targets/releases' delegation + Returns tags, their hashes, and their """ + if not targets_file: + targets_file = 'targets/releases.json' gun = "%s/%s" % (namespace, repository) + try: - response = self._get(gun, "targets/releases.json") - json_response = response.json() - targets = self._parse_targets(json_response) + response = self._get(gun, targets_file) + signed = self._parse_signed(response.json()) + targets = signed.get('targets') + expiration = signed.get('expires') except requests.exceptions.Timeout: logger.exception('Timeout when trying to get metadata for %s', gun) - return None, True + return None, None except requests.exceptions.ConnectionError: logger.exception('Connection error when trying to get metadata for %s', gun) - return None, True + return None, None except (requests.exceptions.RequestException, ValueError): logger.exception('Failed to get metadata for %s', gun) - return None, False + return None, None except Non200ResponseException as ex: - return None, ex.response.status_code != 404 and ex.response.status_code != 400 + logger.exception('Failed request for %s: %s', gun, str(ex)) + return None, None except InvalidMetadataException as ex: logger.exception('Failed to parse targets from metadata', str(ex)) - return None, False + return None, None - return targets, False + return targets, expiration - def _parse_targets(self, json_response): + def _parse_signed(self, json_response): """ Attempts to parse the targets from a metadata response """ signed = json_response.get('signed') if not signed: raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response) - targets = signed.get('targets') - if not targets: - raise InvalidMetadataException("Could not find `targets` in metadata: %s" % json_response) - return targets + return signed def _auth_header(self, gun): """ Generate a registry auth token for apostille""" diff --git a/util/tufmetadata/test/test_tufmetadata.py b/util/tufmetadata/test/test_tufmetadata.py index 98f36ccb5..f46097768 100644 --- a/util/tufmetadata/test/test_tufmetadata.py +++ b/util/tufmetadata/test/test_tufmetadata.py @@ -35,8 +35,8 @@ valid_response = { @pytest.mark.parametrize('response_code,response_body,expected', [ - (200, valid_response, valid_response['signed']['targets']), - (200, {'garbage': 'data'}, None) + (200, valid_response, (valid_response['signed']['targets'], '2020-03-30T18:55:26.594764859-04:00')), + (200, {'garbage': 'data'}, (None, None)) ]) def test_get_metadata(response_code, response_body, expected): app = Flask(__name__) @@ -46,8 +46,8 @@ def test_get_metadata(response_code, response_body, expected): request.json.return_value = response_body client.request.return_value = request tuf_api = api.TUFMetadataAPI(app, app.config, client=client) - tags, _ = tuf_api.get_default_tags('quay', 'quay') - assert tags == expected + response = tuf_api.get_default_tags_with_expiration('quay', 'quay') + assert response == expected @pytest.mark.parametrize('connection_error,response_code,exception', [ (True, 200, requests.exceptions.Timeout), @@ -67,5 +67,6 @@ def test_get_metadata_exception(connection_error, response_code, exception): client = mock.Mock(request=request) client.request.side_effect = exception tuf_api = api.TUFMetadataAPI(app, app.config, client=client) - tags, _ = tuf_api.get_default_tags('quay', 'quay') - assert tags == None \ No newline at end of file + tags, expiration = tuf_api.get_default_tags_with_expiration('quay', 'quay') + assert tags == None + assert expiration == None \ No newline at end of file From c2d7fc2288df77653b6bf77ba892bbe5870c3753 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Fri, 7 Apr 2017 16:14:25 -0400 Subject: [PATCH 4/4] Fix comment --- endpoints/api/signing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/api/signing.py b/endpoints/api/signing.py index 454482850..aa23b7062 100644 --- a/endpoints/api/signing.py +++ b/endpoints/api/signing.py @@ -1,4 +1,4 @@ -""" List and manage repository vulnerabilities and other security information. """ +""" List and manage repository signing information """ import logging import features