Add tufmetadata endpoint
This commit is contained in:
parent
1bfca871ec
commit
9515f18fb6
9 changed files with 282 additions and 1 deletions
2
app.py
2
app.py
|
@ -45,6 +45,7 @@ from util.metrics.metricqueue import MetricQueue
|
|||
from util.metrics.prometheus import PrometheusPlugin
|
||||
from util.saas.cloudwatch import start_cloudwatch_sender
|
||||
from util.secscan.api import SecurityScannerAPI
|
||||
from util.tufmetadata.api import TUFMetadataAPI
|
||||
from util.security.instancekeys import InstanceKeys
|
||||
from util.security.signing import Signer
|
||||
|
||||
|
@ -195,6 +196,7 @@ all_queues = [image_replication_queue, dockerfile_build_queue, notification_queu
|
|||
secscan_notification_queue, chunk_cleanup_queue]
|
||||
|
||||
secscan_api = SecurityScannerAPI(app, app.config, storage)
|
||||
tuf_metadata_api = TUFMetadataAPI(app, app.config)
|
||||
|
||||
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
|
||||
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)
|
||||
|
|
34
endpoints/api/signing.py
Normal file
34
endpoints/api/signing.py
Normal file
|
@ -0,0 +1,34 @@
|
|||
""" List and manage repository vulnerabilities and other security information. """
|
||||
|
||||
import logging
|
||||
import features
|
||||
|
||||
from app import tuf_metadata_api
|
||||
from endpoints.api import (require_repo_read, path_param,
|
||||
RepositoryParamResource, resource, nickname, show_if,
|
||||
disallow_for_app_repositories)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _default_signed_tags_for_repository(namespace, repository):
|
||||
""" Fetches the tags in the targets/releases delegation, which is the one the docker client will trust. """
|
||||
tag_data, _ = tuf_metadata_api.get_default_tags(namespace, repository)
|
||||
return {
|
||||
'tags': tag_data.keys()
|
||||
}
|
||||
|
||||
|
||||
@show_if(features.SIGNING)
|
||||
@resource('/v1/repository/<apirepopath:repository>/signatures')
|
||||
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
||||
class RepositorySignatures(RepositoryParamResource):
|
||||
""" Operations for managing the signatures in a repository image. """
|
||||
|
||||
@require_repo_read
|
||||
@nickname('getRepoSignatures')
|
||||
@disallow_for_app_repositories
|
||||
def get(self, namespace, repository):
|
||||
""" Fetches the list of signed tags for the repository"""
|
||||
return _default_signed_tags_for_repository(namespace, repository)
|
||||
|
|
@ -10,6 +10,7 @@ from endpoints.api.repositorynotification import (RepositoryNotification,
|
|||
RepositoryNotificationList,
|
||||
TestRepositoryNotification)
|
||||
from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
|
||||
from endpoints.api.signing import RepositorySignatures
|
||||
from endpoints.api.tag import ListRepositoryTags, RepositoryTag, RepositoryTagImages, RestoreTag
|
||||
from endpoints.api.trigger import (BuildTriggerList, BuildTrigger, BuildTriggerSubdirs,
|
||||
BuildTriggerActivate, BuildTriggerAnalyze, ActivateBuildTrigger,
|
||||
|
@ -48,6 +49,7 @@ FIELD_ARGS = {'trigger_uuid': '1234', 'field_name': 'foobar'}
|
|||
(TestRepositoryNotification, 'post', NOTIFICATION_ARGS),
|
||||
(RepositoryImageSecurity, 'get', IMAGE_ARGS),
|
||||
(RepositoryManifestSecurity, 'get', MANIFEST_ARGS),
|
||||
(RepositorySignatures, 'get', None),
|
||||
(ListRepositoryTags, 'get', None),
|
||||
(RepositoryTag, 'put', TAG_ARGS),
|
||||
(RepositoryTag, 'delete', TAG_ARGS),
|
||||
|
|
|
@ -64,7 +64,7 @@ def generate_registry_jwt(auth_result):
|
|||
user_event_data = {
|
||||
'action': 'login',
|
||||
}
|
||||
tuf_root = 'quay'
|
||||
tuf_root = QUAY_TUF_ROOT
|
||||
|
||||
if len(scope_param) > 0:
|
||||
match = get_scope_regex().match(scope_param)
|
||||
|
|
|
@ -57,6 +57,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
|
|||
SuperUserRepositoryBuildResource, SuperUserRepositoryBuildStatus)
|
||||
from endpoints.api.globalmessages import GlobalUserMessage, GlobalUserMessages
|
||||
from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
|
||||
from endpoints.api.signing import RepositorySignatures
|
||||
from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
|
||||
|
||||
|
||||
|
@ -4485,6 +4486,24 @@ class TestRepositoryManifestSecurity(ApiTestCase):
|
|||
def test_get_devtable(self):
|
||||
self._run_test('GET', 404, 'devtable', None)
|
||||
|
||||
class TestRepositorySignatures(ApiTestCase):
|
||||
def setUp(self):
|
||||
ApiTestCase.setUp(self)
|
||||
self._set_url(RepositorySignatures, repository='devtable/simple')
|
||||
|
||||
def test_get_anonymous(self):
|
||||
self._run_test('GET', 401, None, None)
|
||||
|
||||
def test_get_freshuser(self):
|
||||
self._run_test('GET', 403, 'freshuser', None)
|
||||
|
||||
def test_get_reader(self):
|
||||
self._run_test('GET', 403, 'reader', None)
|
||||
|
||||
def test_get_devtable(self):
|
||||
self._run_test('GET', 404, 'devtable', None)
|
||||
|
||||
|
||||
class TestRepositoryManifestLabels(ApiTestCase):
|
||||
def setUp(self):
|
||||
ApiTestCase.setUp(self)
|
||||
|
|
|
@ -74,6 +74,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
|
|||
SuperUserCustomCertificates, SuperUserCustomCertificate)
|
||||
from endpoints.api.globalmessages import (GlobalUserMessage, GlobalUserMessages,)
|
||||
from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
|
||||
from endpoints.api.signing import RepositorySignatures
|
||||
from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile,
|
||||
SuperUserCreateInitialSuperUser)
|
||||
from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
|
||||
|
@ -4506,6 +4507,31 @@ class TestRepositoryImageSecurity(ApiTestCase):
|
|||
expected_code=200)
|
||||
|
||||
|
||||
class TestRepositorySignatures(ApiTestCase):
|
||||
def test_get_signatures(self):
|
||||
self.login(ADMIN_ACCESS_USER)
|
||||
|
||||
targets = {
|
||||
'latest': {
|
||||
'hashes': {
|
||||
'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
|
||||
},
|
||||
'length': 1500
|
||||
},
|
||||
'test_tag': {
|
||||
'hashes': {
|
||||
'sha256': '1234123'
|
||||
},
|
||||
'length': 50
|
||||
}
|
||||
}
|
||||
|
||||
with patch('app.tuf_metadata_api') as mock_tuf:
|
||||
mock_tuf.get_default_tags.return_value = targets
|
||||
signed_tags_response = self.getJsonResponse(RepositorySignatures, params=dict(namespace='ns', repository='repo'))
|
||||
self.assertEquals(signed_tags_response, {'tags': ['latest', 'test_tag']})
|
||||
|
||||
|
||||
class TestSuperUserCustomCertificates(ApiTestCase):
|
||||
def test_custom_certificates(self):
|
||||
self.login(ADMIN_ACCESS_USER)
|
||||
|
|
0
util/tufmetadata/__init__.py
Normal file
0
util/tufmetadata/__init__.py
Normal file
127
util/tufmetadata/api.py
Normal file
127
util/tufmetadata/api.py
Normal file
|
@ -0,0 +1,127 @@
|
|||
import logging
|
||||
from urlparse import urljoin
|
||||
|
||||
import requests
|
||||
|
||||
from data.database import CloseForLongOperation
|
||||
from util.failover import failover, FailoverException
|
||||
from util.security.instancekeys import InstanceKeys
|
||||
from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT
|
||||
|
||||
|
||||
DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
|
||||
MITM_CERT_PATH = '/conf/mitm.cert'
|
||||
TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InvalidMetadataException(Exception):
|
||||
""" Exception raised when the upstream API metadata that doesn't parse correctly. """
|
||||
pass
|
||||
|
||||
|
||||
class Non200ResponseException(Exception):
|
||||
""" Exception raised when the upstream API returns a non-200 HTTP status code. """
|
||||
def __init__(self, response):
|
||||
super(Non200ResponseException, self).__init__()
|
||||
self.response = response
|
||||
|
||||
|
||||
class TUFMetadataAPI(object):
|
||||
""" Helper class for talking to the TUF Metadata service (Apostille). """
|
||||
def __init__(self, app, config, client=None):
|
||||
self._app = app
|
||||
self._instance_keys = InstanceKeys(app)
|
||||
self._config = config
|
||||
self._client = client or config['HTTPCLIENT']
|
||||
|
||||
def get_default_tags(self, namespace, repository):
|
||||
""" Gets the tag -> sha mappings in the 'targets/releases' delegation
|
||||
"""
|
||||
gun = "%s/%s" % (namespace, repository)
|
||||
try:
|
||||
response = self._get(gun, "targets/releases.json")
|
||||
json_response = response.json()
|
||||
targets = self._parse_targets(json_response)
|
||||
except requests.exceptions.Timeout:
|
||||
logger.exception('Timeout when trying to get metadata for %s', gun)
|
||||
return None, True
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.exception('Connection error when trying to get metadata for %s', gun)
|
||||
return None, True
|
||||
except (requests.exceptions.RequestException, ValueError):
|
||||
logger.exception('Failed to get metadata for %s', gun)
|
||||
return None, False
|
||||
except Non200ResponseException as ex:
|
||||
return None, ex.response.status_code != 404 and ex.response.status_code != 400
|
||||
except InvalidMetadataException as ex:
|
||||
logger.exception('Failed to parse targets from metadata', str(ex))
|
||||
return None, False
|
||||
|
||||
return targets, False
|
||||
|
||||
def _parse_targets(self, json_response):
|
||||
""" Attempts to parse the targets from a metadata response """
|
||||
signed = json_response.get('signed')
|
||||
if not signed:
|
||||
raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response)
|
||||
targets = signed.get('targets')
|
||||
if not targets:
|
||||
raise InvalidMetadataException("Could not find `targets` in metadata: %s" % json_response)
|
||||
return targets
|
||||
|
||||
def _auth_header(self, gun):
|
||||
""" Generate a registry auth token for apostille"""
|
||||
access = [{
|
||||
'type': 'repository',
|
||||
'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun),
|
||||
'actions': ['pull'],
|
||||
}]
|
||||
context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT)
|
||||
token = generate_bearer_token("quay", subject, context, access,
|
||||
TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
|
||||
return {'Authorization': 'Bearer %s' % token}
|
||||
|
||||
def _get(self, gun, metadata_file):
|
||||
return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun))
|
||||
|
||||
def _request(self, method, endpoint, path, body, headers, params, timeout):
|
||||
""" Issues an HTTP request to the signing endpoint. """
|
||||
url = urljoin(endpoint, path)
|
||||
logger.debug('%sing signing URL %s', method.upper(), url)
|
||||
|
||||
headers.update(DEFAULT_HTTP_HEADERS)
|
||||
resp = self._client.request(method, url, json=body, params=params, timeout=timeout,
|
||||
verify=MITM_CERT_PATH, headers=headers)
|
||||
if resp.status_code // 100 != 2:
|
||||
raise Non200ResponseException(resp)
|
||||
return resp
|
||||
|
||||
def _call(self, method, path, params=None, body=None, headers=None):
|
||||
""" Issues an HTTP request to signing service and handles failover for GET requests.
|
||||
"""
|
||||
if self._config is None:
|
||||
raise Exception('Cannot call unconfigured signing service')
|
||||
|
||||
timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1)
|
||||
endpoint = self._config['TUF_SERVER']
|
||||
|
||||
with CloseForLongOperation(self._config):
|
||||
# If the request isn't a read do not fail over.
|
||||
if method != 'GET':
|
||||
return self._request(method, endpoint, path, body, headers, params, timeout)
|
||||
|
||||
# The request is read-only and can failover.
|
||||
all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', [])
|
||||
return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {})
|
||||
for endpoint in all_endpoints])
|
||||
|
||||
|
||||
@failover
|
||||
def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout):
|
||||
""" This function auto-retries read-only requests until they return a 2xx status code. """
|
||||
try:
|
||||
return request_fn('GET', endpoint, path, body, headers, params, timeout)
|
||||
except (requests.exceptions.RequestException, Non200ResponseException) as ex:
|
||||
raise FailoverException(ex)
|
71
util/tufmetadata/test/test_tufmetadata.py
Normal file
71
util/tufmetadata/test/test_tufmetadata.py
Normal file
|
@ -0,0 +1,71 @@
|
|||
import pytest
|
||||
import requests
|
||||
from mock import mock
|
||||
|
||||
from flask import Flask
|
||||
|
||||
from test import testconfig
|
||||
from util.tufmetadata import api
|
||||
|
||||
valid_response = {
|
||||
'signed' : {
|
||||
'type': 'Targets',
|
||||
'delegations': {
|
||||
'keys': {},
|
||||
'roles': {},
|
||||
},
|
||||
'expires': '2020-03-30T18:55:26.594764859-04:00',
|
||||
'targets': {
|
||||
'latest': {
|
||||
'hashes': {
|
||||
'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
|
||||
},
|
||||
'length': 1500
|
||||
}
|
||||
},
|
||||
'version': 2
|
||||
},
|
||||
'signatures': [
|
||||
{
|
||||
'method': 'ecdsa',
|
||||
'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw=='
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('response_code,response_body,expected', [
|
||||
(200, valid_response, valid_response['signed']['targets']),
|
||||
(200, {'garbage': 'data'}, None)
|
||||
])
|
||||
def test_get_metadata(response_code, response_body, expected):
|
||||
app = Flask(__name__)
|
||||
app.config.from_object(testconfig.TestConfig())
|
||||
client = mock.Mock()
|
||||
request = mock.Mock(status_code=response_code)
|
||||
request.json.return_value = response_body
|
||||
client.request.return_value = request
|
||||
tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
|
||||
tags, _ = tuf_api.get_default_tags('quay', 'quay')
|
||||
assert tags == expected
|
||||
|
||||
@pytest.mark.parametrize('connection_error,response_code,exception', [
|
||||
(True, 200, requests.exceptions.Timeout),
|
||||
(True, 200, requests.exceptions.ConnectionError),
|
||||
(False, 200, requests.exceptions.RequestException),
|
||||
(False, 200, ValueError),
|
||||
(True, 500, api.Non200ResponseException(mock.Mock(status_code=500))),
|
||||
(False, 400, api.Non200ResponseException(mock.Mock(status_code=400))),
|
||||
(False, 404, api.Non200ResponseException(mock.Mock(status_code=404))),
|
||||
(False, 200, api.InvalidMetadataException)
|
||||
|
||||
])
|
||||
def test_get_metadata_exception(connection_error, response_code, exception):
|
||||
app = Flask(__name__)
|
||||
app.config.from_object(testconfig.TestConfig())
|
||||
request = mock.Mock(status_code=response_code)
|
||||
client = mock.Mock(request=request)
|
||||
client.request.side_effect = exception
|
||||
tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
|
||||
tags, _ = tuf_api.get_default_tags('quay', 'quay')
|
||||
assert tags == None
|
Reference in a new issue