Merge pull request #2512 from ecordell/tufmetadata

Add tufmetadata endpoint

app.py

@@ -45,6 +45,7 @@ from util.metrics.metricqueue import MetricQueue
 from util.metrics.prometheus import PrometheusPlugin
 from util.saas.cloudwatch import start_cloudwatch_sender
 from util.secscan.api import SecurityScannerAPI
+from util.tufmetadata.api import TUFMetadataAPI
 from util.security.instancekeys import InstanceKeys
 from util.security.signing import Signer
 
@@ -195,6 +196,7 @@ all_queues = [image_replication_queue, dockerfile_build_queue, notification_queu
               secscan_notification_queue, chunk_cleanup_queue]
 
 secscan_api = SecurityScannerAPI(app, app.config, storage)
+tuf_metadata_api = TUFMetadataAPI(app, app.config)
 
 # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
 _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)

endpoints/api/signing.py

@@ -0,0 +1,30 @@
+""" List and manage repository signing information """
+
+import logging
+import features
+
+from app import tuf_metadata_api
+from endpoints.api import (require_repo_read, path_param,
+                           RepositoryParamResource, resource, nickname, show_if, 
+                           disallow_for_app_repositories)
+
+logger = logging.getLogger(__name__)
+
+
+@show_if(features.SIGNING)
+@resource('/v1/repository/<apirepopath:repository>/signatures')
+@path_param('repository', 'The full path of the repository. e.g. namespace/name')
+class RepositorySignatures(RepositoryParamResource):
+  """ Operations for managing the signatures in a repository image. """
+
+  @require_repo_read
+  @nickname('getRepoSignatures')
+  @disallow_for_app_repositories
+  def get(self, namespace, repository):
+    """ Fetches the list of signed tags for the repository"""
+    tag_data, expiration = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository)
+    return {
+      'tags': tag_data,
+      'expiration': expiration
+    }
+

endpoints/api/test/test_disallow_for_apps.py

@@ -10,6 +10,7 @@ from endpoints.api.repositorynotification import (RepositoryNotification,
                                                   RepositoryNotificationList,
                                                   TestRepositoryNotification)
 from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
+from endpoints.api.signing import RepositorySignatures
 from endpoints.api.tag import ListRepositoryTags, RepositoryTag, RepositoryTagImages, RestoreTag
 from endpoints.api.trigger import (BuildTriggerList, BuildTrigger, BuildTriggerSubdirs,
                                    BuildTriggerActivate, BuildTriggerAnalyze, ActivateBuildTrigger,
@@ -47,6 +48,7 @@ FIELD_ARGS = {'trigger_uuid': '1234', 'field_name': 'foobar'}
   (TestRepositoryNotification, 'post', NOTIFICATION_ARGS),
   (RepositoryImageSecurity, 'get', IMAGE_ARGS),
   (RepositoryManifestSecurity, 'get', MANIFEST_ARGS),
+  (RepositorySignatures, 'get', None),
   (ListRepositoryTags, 'get', None),
   (RepositoryTag, 'put', TAG_ARGS),
   (RepositoryTag, 'delete', TAG_ARGS),

endpoints/api/test/test_security.py

@@ -1,14 +1,17 @@
 import pytest
+from flask_principal import AnonymousIdentity
 
 from endpoints.api import api
 from endpoints.api.team import OrganizationTeamSyncing
 from endpoints.api.test.shared import client_with_identity, conduct_api_call
 from endpoints.api.superuser import SuperUserRepositoryBuildLogs, SuperUserRepositoryBuildResource
 from endpoints.api.superuser import SuperUserRepositoryBuildStatus
+from endpoints.api.signing import RepositorySignatures
 from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file
 
 TEAM_PARAMS = {'orgname': 'buynlarge', 'teamname': 'owners'}
 BUILD_PARAMS = {'build_uuid': 'test-1234'}
+REPO_PARAMS = {'repository': 'devtable/someapp'}
 
 @pytest.mark.parametrize('resource,method,params,body,identity,expected', [
   (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, None, 403),
@@ -35,6 +38,10 @@ BUILD_PARAMS = {'build_uuid': 'test-1234'}
   (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'freshuser', 403),
   (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None, 'reader', 403),
   (SuperUserRepositoryBuildResource, 'GET', BUILD_PARAMS, None,  'devtable', 404),
+  
+  (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'freshuser', 403),
+  (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'reader', 403),
+  (RepositorySignatures, 'GET', REPO_PARAMS, {}, 'devtable', 200),
 ])
 def test_api_security(resource, method, params, body, identity, expected, client):
   with client_with_identity(identity, client) as cl:

endpoints/api/test/test_signing.py

@@ -0,0 +1,43 @@
+from collections import Counter
+
+import pytest
+
+from endpoints.api.test.shared import client_with_identity, conduct_api_call
+from endpoints.api.signing import RepositorySignatures
+from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file
+from mock import patch
+
+VALID_TARGETS = {
+  'latest': {
+    'hashes': {
+      'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
+    },
+    'length': 1500
+  },
+  'test_tag': {
+    'hashes': {
+      'sha256': '1234123'
+    },
+    'length': 50
+  }
+}
+
+def tags_equal(expected, actual):
+  expected_tags = expected.get('tags')
+  actual_tags = actual.get('tags')
+  if expected_tags and actual_tags:
+    return Counter(expected_tags) == Counter(actual_tags)
+  return expected == actual
+
+@pytest.mark.parametrize('targets,expected', [
+  (VALID_TARGETS, {'tags': VALID_TARGETS, 'expiration': 'expires'}), 
+  ({'bad': 'tags'}, {'tags': {'bad': 'tags'}, 'expiration': 'expires'}),
+  ({}, {'tags': {}, 'expiration': 'expires'}),
+  (None, {'tags': None, 'expiration': 'expires'}),   # API returns None on exceptions
+])
+def test_get_signatures(targets, expected, client):
+  with patch('endpoints.api.signing.tuf_metadata_api') as mock_tuf:
+    mock_tuf.get_default_tags_with_expiration.return_value = (targets, 'expires')
+    with client_with_identity('devtable', client) as cl:
+      params = {'repository': 'devtable/repo'}
+      assert tags_equal(expected, conduct_api_call(cl, RepositorySignatures, 'GET', params, None, 200).json)

endpoints/v2/v2auth.py

@@ -64,7 +64,7 @@ def generate_registry_jwt(auth_result):
   user_event_data = {
     'action': 'login',
   }
-  tuf_root = 'quay'
+  tuf_root = QUAY_TUF_ROOT
 
   if len(scope_param) > 0:
     match = get_scope_regex().match(scope_param)

test/test_api_security.py

@@ -4485,6 +4485,7 @@ class TestRepositoryManifestSecurity(ApiTestCase):
   def test_get_devtable(self):
     self._run_test('GET', 404, 'devtable', None)
 
+
 class TestRepositoryManifestLabels(ApiTestCase):
   def setUp(self):
     ApiTestCase.setUp(self)

test/test_api_usage.py

@@ -4540,6 +4540,8 @@ class TestRepositoryImageSecurity(ApiTestCase):
                          expected_code=200)
 
 
+      
+
 class TestSuperUserCustomCertificates(ApiTestCase):
   def test_custom_certificates(self):
     self.login(ADMIN_ACCESS_USER)

test/testconfig.py

@@ -65,6 +65,8 @@ class TestConfig(DefaultConfig):
   SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1
   SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1
   
+  FEATURE_SIGNING = True
+
   SIGNING_ENGINE = 'gpg2'
 
   GPG2_PRIVATE_KEY_NAME = 'EEB32221'

util/tufmetadata/api.py

@@ -0,0 +1,130 @@
+import logging
+from urlparse import urljoin
+
+import requests
+
+from data.database import CloseForLongOperation
+from util.failover import failover, FailoverException
+from util.security.instancekeys import InstanceKeys
+from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT
+
+
+DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
+MITM_CERT_PATH = '/conf/mitm.cert'
+TOKEN_VALIDITY_LIFETIME_S = 60 * 60  # 1 hour
+
+logger = logging.getLogger(__name__)
+
+
+class InvalidMetadataException(Exception):
+  """ Exception raised when the upstream API metadata that doesn't parse correctly. """
+  pass
+
+
+class Non200ResponseException(Exception):
+  """ Exception raised when the upstream API returns a non-200 HTTP status code. """
+  def __init__(self, response):
+    super(Non200ResponseException, self).__init__()
+    self.response = response
+
+
+class TUFMetadataAPI(object):
+  """ Helper class for talking to the TUF Metadata service (Apostille). """
+  def __init__(self, app, config, client=None):
+    self._app = app
+    self._instance_keys = InstanceKeys(app)
+    self._config = config
+    self._client = client or config['HTTPCLIENT']
+
+  def get_default_tags_with_expiration(self, namespace, repository, targets_file=None):
+    """ Gets the tag -> sha mappings in the 'targets/releases' delegation
+        Returns tags, their hashes, and their 
+    """
+    if not targets_file:
+      targets_file = 'targets/releases.json'
+    gun = "%s/%s" % (namespace, repository)
+    
+    try:
+      response = self._get(gun, targets_file)
+      signed = self._parse_signed(response.json())
+      targets = signed.get('targets')
+      expiration = signed.get('expires')
+    except requests.exceptions.Timeout:
+      logger.exception('Timeout when trying to get metadata for %s', gun)
+      return None, None 
+    except requests.exceptions.ConnectionError:
+      logger.exception('Connection error when trying to get metadata for %s', gun)
+      return None, None 
+    except (requests.exceptions.RequestException, ValueError):
+      logger.exception('Failed to get metadata for %s', gun)
+      return None, None 
+    except Non200ResponseException as ex:
+      logger.exception('Failed request for %s: %s', gun, str(ex))
+      return None, None 
+    except InvalidMetadataException as ex:
+      logger.exception('Failed to parse targets from metadata', str(ex))
+      return None, None 
+    
+    return targets, expiration
+  
+  def _parse_signed(self, json_response):
+    """ Attempts to parse the targets from a metadata response """
+    signed = json_response.get('signed')
+    if not signed:
+      raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response)
+    return signed 
+  
+  def _auth_header(self, gun):
+    """ Generate a registry auth token for apostille"""
+    access = [{
+      'type': 'repository',
+      'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun),
+      'actions': ['pull'],
+    }]
+    context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT)
+    token = generate_bearer_token("quay", subject, context, access,
+                                  TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
+    return {'Authorization': 'Bearer %s' % token}
+    
+  def _get(self, gun, metadata_file):
+    return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun))
+    
+  def _request(self, method, endpoint, path, body, headers, params, timeout):
+    """ Issues an HTTP request to the signing endpoint. """
+    url = urljoin(endpoint, path)
+    logger.debug('%sing signing URL %s', method.upper(), url)
+
+    headers.update(DEFAULT_HTTP_HEADERS)    
+    resp = self._client.request(method, url, json=body, params=params, timeout=timeout,
+                                verify=MITM_CERT_PATH, headers=headers)
+    if resp.status_code // 100 != 2:
+      raise Non200ResponseException(resp)
+    return resp
+
+  def _call(self, method, path, params=None, body=None, headers=None):
+    """ Issues an HTTP request to signing service and handles failover for GET requests.
+    """
+    if self._config is None:
+      raise Exception('Cannot call unconfigured signing service')
+
+    timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1)
+    endpoint = self._config['TUF_SERVER']
+
+    with CloseForLongOperation(self._config):
+      # If the request isn't a read do not fail over.
+      if method != 'GET':
+        return self._request(method, endpoint, path, body, headers, params, timeout)
+
+      # The request is read-only and can failover.
+      all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', [])
+      return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {})
+                                      for endpoint in all_endpoints])
+
+
+@failover
+def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout):
+  """ This function auto-retries read-only requests until they return a 2xx status code. """
+  try:
+    return request_fn('GET', endpoint, path, body, headers, params, timeout)
+  except (requests.exceptions.RequestException, Non200ResponseException) as ex:
+    raise FailoverException(ex)

util/tufmetadata/test/test_tufmetadata.py

@@ -0,0 +1,72 @@
+import pytest
+import requests
+from mock import mock
+
+from flask import Flask
+
+from test import testconfig
+from util.tufmetadata import api 
+
+valid_response = {
+  'signed' : {
+    'type': 'Targets',
+    'delegations': {
+      'keys': {},
+      'roles': {},
+    },
+    'expires': '2020-03-30T18:55:26.594764859-04:00',
+    'targets': {
+      'latest': {
+        'hashes': {
+          'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
+        },
+        'length': 1500
+      }
+    },
+    'version': 2
+  },
+  'signatures': [
+    {
+      'method': 'ecdsa',
+      'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw=='
+    }
+  ]
+}
+
+
+@pytest.mark.parametrize('response_code,response_body,expected', [
+  (200, valid_response, (valid_response['signed']['targets'], '2020-03-30T18:55:26.594764859-04:00')),
+  (200, {'garbage': 'data'}, (None, None))
+])
+def test_get_metadata(response_code, response_body, expected): 
+    app = Flask(__name__) 
+    app.config.from_object(testconfig.TestConfig())
+    client = mock.Mock()
+    request = mock.Mock(status_code=response_code)
+    request.json.return_value = response_body
+    client.request.return_value = request
+    tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
+    response = tuf_api.get_default_tags_with_expiration('quay', 'quay')
+    assert response == expected 
+    
+@pytest.mark.parametrize('connection_error,response_code,exception', [
+  (True, 200, requests.exceptions.Timeout),
+  (True, 200, requests.exceptions.ConnectionError),
+  (False, 200, requests.exceptions.RequestException),
+  (False, 200, ValueError),
+  (True, 500, api.Non200ResponseException(mock.Mock(status_code=500))),
+  (False, 400, api.Non200ResponseException(mock.Mock(status_code=400))),
+  (False, 404, api.Non200ResponseException(mock.Mock(status_code=404))),
+  (False, 200, api.InvalidMetadataException)
+  
+])
+def test_get_metadata_exception(connection_error, response_code, exception): 
+    app = Flask(__name__) 
+    app.config.from_object(testconfig.TestConfig())
+    request = mock.Mock(status_code=response_code)
+    client = mock.Mock(request=request)
+    client.request.side_effect = exception
+    tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
+    tags, expiration = tuf_api.get_default_tags_with_expiration('quay', 'quay')
+    assert tags == None
+    assert expiration == None