diff --git a/app.py b/app.py index e168f94ba..587432e47 100644 --- a/app.py +++ b/app.py @@ -45,6 +45,7 @@ from util.metrics.metricqueue import MetricQueue from util.metrics.prometheus import PrometheusPlugin from util.saas.cloudwatch import start_cloudwatch_sender from util.secscan.api import SecurityScannerAPI +from util.tufmetadata.api import TUFMetadataAPI from util.security.instancekeys import InstanceKeys from util.security.signing import Signer @@ -195,6 +196,7 @@ all_queues = [image_replication_queue, dockerfile_build_queue, notification_queu secscan_notification_queue, chunk_cleanup_queue] secscan_api = SecurityScannerAPI(app, app.config, storage) +tuf_metadata_api = TUFMetadataAPI(app, app.config) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) diff --git a/endpoints/api/signing.py b/endpoints/api/signing.py new file mode 100644 index 000000000..aa23b7062 --- /dev/null +++ b/endpoints/api/signing.py @@ -0,0 +1,30 @@ +""" List and manage repository signing information """ + +import logging +import features + +from app import tuf_metadata_api +from endpoints.api import (require_repo_read, path_param, + RepositoryParamResource, resource, nickname, show_if, + disallow_for_app_repositories) + +logger = logging.getLogger(__name__) + + +@show_if(features.SIGNING) +@resource('/v1/repository//signatures') +@path_param('repository', 'The full path of the repository. e.g. namespace/name') +class RepositorySignatures(RepositoryParamResource): + """ Operations for managing the signatures in a repository image. """ + + @require_repo_read + @nickname('getRepoSignatures') + @disallow_for_app_repositories + def get(self, namespace, repository): + """ Fetches the list of signed tags for the repository""" + tag_data, expiration = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository) + return { + 'tags': tag_data, + 'expiration': expiration + } + diff --git a/endpoints/api/test/test_disallow_for_apps.py b/endpoints/api/test/test_disallow_for_apps.py index 59fb2554d..de695172a 100644 --- a/endpoints/api/test/test_disallow_for_apps.py +++ b/endpoints/api/test/test_disallow_for_apps.py @@ -10,6 +10,7 @@ from endpoints.api.repositorynotification import (RepositoryNotification, RepositoryNotificationList, TestRepositoryNotification) from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity +from endpoints.api.signing import RepositorySignatures from endpoints.api.tag import ListRepositoryTags, RepositoryTag, RepositoryTagImages, RestoreTag from endpoints.api.trigger import (BuildTriggerList, BuildTrigger, BuildTriggerSubdirs, BuildTriggerActivate, BuildTriggerAnalyze, ActivateBuildTrigger, @@ -47,6 +48,7 @@ FIELD_ARGS = {'trigger_uuid': '1234', 'field_name': 'foobar'} (TestRepositoryNotification, 'post', NOTIFICATION_ARGS), (RepositoryImageSecurity, 'get', IMAGE_ARGS), (RepositoryManifestSecurity, 'get', MANIFEST_ARGS), + (RepositorySignatures, 'get', None), (ListRepositoryTags, 'get', None), (RepositoryTag, 'put', TAG_ARGS), (RepositoryTag, 'delete', TAG_ARGS), diff --git a/endpoints/api/test/test_security.py b/endpoints/api/test/test_security.py index 1ae2c31cc..425ad1682 100644 --- a/endpoints/api/test/test_security.py +++ b/endpoints/api/test/test_security.py @@ -1,14 +1,17 @@ import pytest +from flask_principal import AnonymousIdentity from endpoints.api import api from endpoints.api.team import OrganizationTeamSyncing from endpoints.api.test.shared import client_with_identity, conduct_api_call from endpoints.api.superuser import SuperUserRepositoryBuildLogs, SuperUserRepositoryBuildResource from endpoints.api.superuser import SuperUserRepositoryBuildStatus +from endpoints.api.signing import RepositorySignatures from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file TEAM_PARAMS = {'orgname': 'buynlarge', 'teamname': 'owners'} BUILD_PARAMS = {'build_uuid': 'test-1234'} +REPO_PARAMS = {'repository': 'devtable/someapp'} @pytest.mark.parametrize('resource,method,params,body,identity,expected', [ (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, None, 403), @@ -35,6 +38,10 @@ BUILD_PARAMS = {'build_uuid': 'test-1234'} (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'freshuser', 403), (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'reader', 403), (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'devtable', 404), + + (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'freshuser', 403), + (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'reader', 403), + (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'devtable', 200), ]) def test_api_security(resource, method, params, body, identity, expected, client): with client_with_identity(identity, client) as cl: diff --git a/endpoints/api/test/test_signing.py b/endpoints/api/test/test_signing.py new file mode 100644 index 000000000..93ac94be6 --- /dev/null +++ b/endpoints/api/test/test_signing.py @@ -0,0 +1,43 @@ +from collections import Counter + +import pytest + +from endpoints.api.test.shared import client_with_identity, conduct_api_call +from endpoints.api.signing import RepositorySignatures +from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file +from mock import patch + +VALID_TARGETS = { + 'latest': { + 'hashes': { + 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' + }, + 'length': 1500 + }, + 'test_tag': { + 'hashes': { + 'sha256': '1234123' + }, + 'length': 50 + } +} + +def tags_equal(expected, actual): + expected_tags = expected.get('tags') + actual_tags = actual.get('tags') + if expected_tags and actual_tags: + return Counter(expected_tags) == Counter(actual_tags) + return expected == actual + +@pytest.mark.parametrize('targets,expected', [ + (VALID_TARGETS, {'tags': VALID_TARGETS, 'expiration': 'expires'}), + ({'bad': 'tags'}, {'tags': {'bad': 'tags'}, 'expiration': 'expires'}), + ({}, {'tags': {}, 'expiration': 'expires'}), + (None, {'tags': None, 'expiration': 'expires'}), # API returns None on exceptions +]) +def test_get_signatures(targets, expected, client): + with patch('endpoints.api.signing.tuf_metadata_api') as mock_tuf: + mock_tuf.get_default_tags_with_expiration.return_value = (targets, 'expires') + with client_with_identity('devtable', client) as cl: + params = {'repository': 'devtable/repo'} + assert tags_equal(expected, conduct_api_call(cl, RepositorySignatures, 'GET', params, None, 200).json) diff --git a/endpoints/v2/v2auth.py b/endpoints/v2/v2auth.py index c304300d4..12e7863a8 100644 --- a/endpoints/v2/v2auth.py +++ b/endpoints/v2/v2auth.py @@ -64,7 +64,7 @@ def generate_registry_jwt(auth_result): user_event_data = { 'action': 'login', } - tuf_root = 'quay' + tuf_root = QUAY_TUF_ROOT if len(scope_param) > 0: match = get_scope_regex().match(scope_param) diff --git a/test/test_api_security.py b/test/test_api_security.py index 6cd2fbdd4..0df692b17 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -4485,6 +4485,7 @@ class TestRepositoryManifestSecurity(ApiTestCase): def test_get_devtable(self): self._run_test('GET', 404, 'devtable', None) + class TestRepositoryManifestLabels(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 4e2f6823f..8afe5f5a6 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -4540,6 +4540,8 @@ class TestRepositoryImageSecurity(ApiTestCase): expected_code=200) + + class TestSuperUserCustomCertificates(ApiTestCase): def test_custom_certificates(self): self.login(ADMIN_ACCESS_USER) diff --git a/test/testconfig.py b/test/testconfig.py index 72c5fb229..ab2a238b9 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -64,6 +64,8 @@ class TestConfig(DefaultConfig): SECURITY_SCANNER_API_VERSION = 'v1' SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1 SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1 + + FEATURE_SIGNING = True SIGNING_ENGINE = 'gpg2' diff --git a/util/tufmetadata/__init__.py b/util/tufmetadata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/tufmetadata/api.py b/util/tufmetadata/api.py new file mode 100644 index 000000000..2ece0c307 --- /dev/null +++ b/util/tufmetadata/api.py @@ -0,0 +1,130 @@ +import logging +from urlparse import urljoin + +import requests + +from data.database import CloseForLongOperation +from util.failover import failover, FailoverException +from util.security.instancekeys import InstanceKeys +from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT + + +DEFAULT_HTTP_HEADERS = {'Connection': 'close'} +MITM_CERT_PATH = '/conf/mitm.cert' +TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour + +logger = logging.getLogger(__name__) + + +class InvalidMetadataException(Exception): + """ Exception raised when the upstream API metadata that doesn't parse correctly. """ + pass + + +class Non200ResponseException(Exception): + """ Exception raised when the upstream API returns a non-200 HTTP status code. """ + def __init__(self, response): + super(Non200ResponseException, self).__init__() + self.response = response + + +class TUFMetadataAPI(object): + """ Helper class for talking to the TUF Metadata service (Apostille). """ + def __init__(self, app, config, client=None): + self._app = app + self._instance_keys = InstanceKeys(app) + self._config = config + self._client = client or config['HTTPCLIENT'] + + def get_default_tags_with_expiration(self, namespace, repository, targets_file=None): + """ Gets the tag -> sha mappings in the 'targets/releases' delegation + Returns tags, their hashes, and their + """ + if not targets_file: + targets_file = 'targets/releases.json' + gun = "%s/%s" % (namespace, repository) + + try: + response = self._get(gun, targets_file) + signed = self._parse_signed(response.json()) + targets = signed.get('targets') + expiration = signed.get('expires') + except requests.exceptions.Timeout: + logger.exception('Timeout when trying to get metadata for %s', gun) + return None, None + except requests.exceptions.ConnectionError: + logger.exception('Connection error when trying to get metadata for %s', gun) + return None, None + except (requests.exceptions.RequestException, ValueError): + logger.exception('Failed to get metadata for %s', gun) + return None, None + except Non200ResponseException as ex: + logger.exception('Failed request for %s: %s', gun, str(ex)) + return None, None + except InvalidMetadataException as ex: + logger.exception('Failed to parse targets from metadata', str(ex)) + return None, None + + return targets, expiration + + def _parse_signed(self, json_response): + """ Attempts to parse the targets from a metadata response """ + signed = json_response.get('signed') + if not signed: + raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response) + return signed + + def _auth_header(self, gun): + """ Generate a registry auth token for apostille""" + access = [{ + 'type': 'repository', + 'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun), + 'actions': ['pull'], + }] + context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT) + token = generate_bearer_token("quay", subject, context, access, + TOKEN_VALIDITY_LIFETIME_S, self._instance_keys) + return {'Authorization': 'Bearer %s' % token} + + def _get(self, gun, metadata_file): + return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun)) + + def _request(self, method, endpoint, path, body, headers, params, timeout): + """ Issues an HTTP request to the signing endpoint. """ + url = urljoin(endpoint, path) + logger.debug('%sing signing URL %s', method.upper(), url) + + headers.update(DEFAULT_HTTP_HEADERS) + resp = self._client.request(method, url, json=body, params=params, timeout=timeout, + verify=MITM_CERT_PATH, headers=headers) + if resp.status_code // 100 != 2: + raise Non200ResponseException(resp) + return resp + + def _call(self, method, path, params=None, body=None, headers=None): + """ Issues an HTTP request to signing service and handles failover for GET requests. + """ + if self._config is None: + raise Exception('Cannot call unconfigured signing service') + + timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1) + endpoint = self._config['TUF_SERVER'] + + with CloseForLongOperation(self._config): + # If the request isn't a read do not fail over. + if method != 'GET': + return self._request(method, endpoint, path, body, headers, params, timeout) + + # The request is read-only and can failover. + all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', []) + return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {}) + for endpoint in all_endpoints]) + + +@failover +def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout): + """ This function auto-retries read-only requests until they return a 2xx status code. """ + try: + return request_fn('GET', endpoint, path, body, headers, params, timeout) + except (requests.exceptions.RequestException, Non200ResponseException) as ex: + raise FailoverException(ex) diff --git a/util/tufmetadata/test/test_tufmetadata.py b/util/tufmetadata/test/test_tufmetadata.py new file mode 100644 index 000000000..f46097768 --- /dev/null +++ b/util/tufmetadata/test/test_tufmetadata.py @@ -0,0 +1,72 @@ +import pytest +import requests +from mock import mock + +from flask import Flask + +from test import testconfig +from util.tufmetadata import api + +valid_response = { + 'signed' : { + 'type': 'Targets', + 'delegations': { + 'keys': {}, + 'roles': {}, + }, + 'expires': '2020-03-30T18:55:26.594764859-04:00', + 'targets': { + 'latest': { + 'hashes': { + 'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo=' + }, + 'length': 1500 + } + }, + 'version': 2 + }, + 'signatures': [ + { + 'method': 'ecdsa', + 'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw==' + } + ] +} + + +@pytest.mark.parametrize('response_code,response_body,expected', [ + (200, valid_response, (valid_response['signed']['targets'], '2020-03-30T18:55:26.594764859-04:00')), + (200, {'garbage': 'data'}, (None, None)) +]) +def test_get_metadata(response_code, response_body, expected): + app = Flask(__name__) + app.config.from_object(testconfig.TestConfig()) + client = mock.Mock() + request = mock.Mock(status_code=response_code) + request.json.return_value = response_body + client.request.return_value = request + tuf_api = api.TUFMetadataAPI(app, app.config, client=client) + response = tuf_api.get_default_tags_with_expiration('quay', 'quay') + assert response == expected + +@pytest.mark.parametrize('connection_error,response_code,exception', [ + (True, 200, requests.exceptions.Timeout), + (True, 200, requests.exceptions.ConnectionError), + (False, 200, requests.exceptions.RequestException), + (False, 200, ValueError), + (True, 500, api.Non200ResponseException(mock.Mock(status_code=500))), + (False, 400, api.Non200ResponseException(mock.Mock(status_code=400))), + (False, 404, api.Non200ResponseException(mock.Mock(status_code=404))), + (False, 200, api.InvalidMetadataException) + +]) +def test_get_metadata_exception(connection_error, response_code, exception): + app = Flask(__name__) + app.config.from_object(testconfig.TestConfig()) + request = mock.Mock(status_code=response_code) + client = mock.Mock(request=request) + client.request.side_effect = exception + tuf_api = api.TUFMetadataAPI(app, app.config, client=client) + tags, expiration = tuf_api.get_default_tags_with_expiration('quay', 'quay') + assert tags == None + assert expiration == None \ No newline at end of file