Add tufmetadata endpoint

app.py

@@ -45,6 +45,7 @@ from util.metrics.metricqueue import MetricQueue
 from util.metrics.prometheus import PrometheusPlugin
 from util.saas.cloudwatch import start_cloudwatch_sender
 from util.secscan.api import SecurityScannerAPI
+from util.tufmetadata.api import TUFMetadataAPI
 from util.security.instancekeys import InstanceKeys
 from util.security.signing import Signer
 
@@ -195,6 +196,7 @@ all_queues = [image_replication_queue, dockerfile_build_queue, notification_queu
               secscan_notification_queue, chunk_cleanup_queue]
 
 secscan_api = SecurityScannerAPI(app, app.config, storage)
+tuf_metadata_api = TUFMetadataAPI(app, app.config)
 
 # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
 _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)

endpoints/api/signing.py

@@ -0,0 +1,34 @@
+""" List and manage repository vulnerabilities and other security information. """
+
+import logging
+import features
+
+from app import tuf_metadata_api
+from endpoints.api import (require_repo_read, path_param,
+                           RepositoryParamResource, resource, nickname, show_if, 
+                           disallow_for_app_repositories)
+
+logger = logging.getLogger(__name__)
+
+
+def _default_signed_tags_for_repository(namespace, repository):
+  """ Fetches the tags in the targets/releases delegation, which is the one the docker client will trust. """
+  tag_data, _ = tuf_metadata_api.get_default_tags(namespace, repository)
+  return {
+    'tags': tag_data.keys()
+  }
+
+
+@show_if(features.SIGNING)
+@resource('/v1/repository/<apirepopath:repository>/signatures')
+@path_param('repository', 'The full path of the repository. e.g. namespace/name')
+class RepositorySignatures(RepositoryParamResource):
+  """ Operations for managing the signatures in a repository image. """
+
+  @require_repo_read
+  @nickname('getRepoSignatures')
+  @disallow_for_app_repositories
+  def get(self, namespace, repository):
+    """ Fetches the list of signed tags for the repository""" 
+    return _default_signed_tags_for_repository(namespace, repository)
+

endpoints/api/test/test_disallow_for_apps.py

@@ -10,6 +10,7 @@ from endpoints.api.repositorynotification import (RepositoryNotification,
                                                   RepositoryNotificationList,
                                                   TestRepositoryNotification)
 from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
+from endpoints.api.signing import RepositorySignatures
 from endpoints.api.tag import ListRepositoryTags, RepositoryTag, RepositoryTagImages, RestoreTag
 from endpoints.api.trigger import (BuildTriggerList, BuildTrigger, BuildTriggerSubdirs,
                                    BuildTriggerActivate, BuildTriggerAnalyze, ActivateBuildTrigger,
@@ -48,6 +49,7 @@ FIELD_ARGS = {'trigger_uuid': '1234', 'field_name': 'foobar'}
   (TestRepositoryNotification, 'post', NOTIFICATION_ARGS),
   (RepositoryImageSecurity, 'get', IMAGE_ARGS),
   (RepositoryManifestSecurity, 'get', MANIFEST_ARGS),
+  (RepositorySignatures, 'get', None),
   (ListRepositoryTags, 'get', None),
   (RepositoryTag, 'put', TAG_ARGS),
   (RepositoryTag, 'delete', TAG_ARGS),

endpoints/v2/v2auth.py

@@ -64,7 +64,7 @@ def generate_registry_jwt(auth_result):
   user_event_data = {
     'action': 'login',
   }
-  tuf_root = 'quay'
+  tuf_root = QUAY_TUF_ROOT
 
   if len(scope_param) > 0:
     match = get_scope_regex().match(scope_param)

test/test_api_security.py

@@ -57,6 +57,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
                                      SuperUserRepositoryBuildResource, SuperUserRepositoryBuildStatus)
 from endpoints.api.globalmessages import GlobalUserMessage, GlobalUserMessages
 from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
+from endpoints.api.signing import RepositorySignatures
 from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
 
 
@@ -4484,6 +4485,24 @@ class TestRepositoryManifestSecurity(ApiTestCase):
 
   def test_get_devtable(self):
     self._run_test('GET', 404, 'devtable', None)
+    
+class TestRepositorySignatures(ApiTestCase):
+  def setUp(self):
+    ApiTestCase.setUp(self)
+    self._set_url(RepositorySignatures, repository='devtable/simple')
+
+  def test_get_anonymous(self):
+    self._run_test('GET', 401, None, None)
+
+  def test_get_freshuser(self):
+    self._run_test('GET', 403, 'freshuser', None)
+
+  def test_get_reader(self):
+    self._run_test('GET', 403, 'reader', None)
+
+  def test_get_devtable(self):
+    self._run_test('GET', 404, 'devtable', None)
+
 
 class TestRepositoryManifestLabels(ApiTestCase):
   def setUp(self):

test/test_api_usage.py

@@ -74,6 +74,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
                                      SuperUserCustomCertificates, SuperUserCustomCertificate)
 from endpoints.api.globalmessages import (GlobalUserMessage, GlobalUserMessages,)
 from endpoints.api.secscan import RepositoryImageSecurity, RepositoryManifestSecurity
+from endpoints.api.signing import RepositorySignatures
 from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile,
                                     SuperUserCreateInitialSuperUser)
 from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
@@ -4506,6 +4507,31 @@ class TestRepositoryImageSecurity(ApiTestCase):
                          expected_code=200)
 
 
+class TestRepositorySignatures(ApiTestCase):
+  def test_get_signatures(self):
+    self.login(ADMIN_ACCESS_USER)
+    
+    targets = {
+      'latest': {
+        'hashes': {
+          'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
+        },
+        'length': 1500
+      },
+      'test_tag': {
+        'hashes': {
+          'sha256': '1234123'
+        },
+        'length': 50
+      }
+    }
+    
+    with patch('app.tuf_metadata_api') as mock_tuf: 
+      mock_tuf.get_default_tags.return_value = targets
+      signed_tags_response = self.getJsonResponse(RepositorySignatures, params=dict(namespace='ns', repository='repo'))
+      self.assertEquals(signed_tags_response, {'tags': ['latest', 'test_tag']})
+      
+
 class TestSuperUserCustomCertificates(ApiTestCase):
   def test_custom_certificates(self):
     self.login(ADMIN_ACCESS_USER)

util/tufmetadata/api.py

@@ -0,0 +1,127 @@
+import logging
+from urlparse import urljoin
+
+import requests
+
+from data.database import CloseForLongOperation
+from util.failover import failover, FailoverException
+from util.security.instancekeys import InstanceKeys
+from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT
+
+
+DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
+MITM_CERT_PATH = '/conf/mitm.cert'
+TOKEN_VALIDITY_LIFETIME_S = 60 * 60  # 1 hour
+
+logger = logging.getLogger(__name__)
+
+
+class InvalidMetadataException(Exception):
+  """ Exception raised when the upstream API metadata that doesn't parse correctly. """
+  pass
+
+
+class Non200ResponseException(Exception):
+  """ Exception raised when the upstream API returns a non-200 HTTP status code. """
+  def __init__(self, response):
+    super(Non200ResponseException, self).__init__()
+    self.response = response
+
+
+class TUFMetadataAPI(object):
+  """ Helper class for talking to the TUF Metadata service (Apostille). """
+  def __init__(self, app, config, client=None):
+    self._app = app
+    self._instance_keys = InstanceKeys(app)
+    self._config = config
+    self._client = client or config['HTTPCLIENT']
+
+  def get_default_tags(self, namespace, repository):
+    """ Gets the tag -> sha mappings in the 'targets/releases' delegation 
+    """
+    gun = "%s/%s" % (namespace, repository)
+    try:
+      response = self._get(gun, "targets/releases.json")
+      json_response = response.json()
+      targets = self._parse_targets(json_response)
+    except requests.exceptions.Timeout:
+      logger.exception('Timeout when trying to get metadata for %s', gun)
+      return None, True
+    except requests.exceptions.ConnectionError:
+      logger.exception('Connection error when trying to get metadata for %s', gun)
+      return None, True
+    except (requests.exceptions.RequestException, ValueError):
+      logger.exception('Failed to get metadata for %s', gun)
+      return None, False
+    except Non200ResponseException as ex:
+      return None, ex.response.status_code != 404 and ex.response.status_code != 400
+    except InvalidMetadataException as ex:
+      logger.exception('Failed to parse targets from metadata', str(ex))
+      return None, False 
+    
+    return targets, False
+  
+  def _parse_targets(self, json_response):
+    """ Attempts to parse the targets from a metadata response """
+    signed = json_response.get('signed')
+    if not signed:
+      raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response)
+    targets = signed.get('targets')
+    if not targets:
+      raise InvalidMetadataException("Could not find `targets` in metadata: %s" % json_response)
+    return targets
+  
+  def _auth_header(self, gun):
+    """ Generate a registry auth token for apostille"""
+    access = [{
+      'type': 'repository',
+      'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun),
+      'actions': ['pull'],
+    }]
+    context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT)
+    token = generate_bearer_token("quay", subject, context, access,
+                                  TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
+    return {'Authorization': 'Bearer %s' % token}
+    
+  def _get(self, gun, metadata_file):
+    return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun))
+    
+  def _request(self, method, endpoint, path, body, headers, params, timeout):
+    """ Issues an HTTP request to the signing endpoint. """
+    url = urljoin(endpoint, path)
+    logger.debug('%sing signing URL %s', method.upper(), url)
+
+    headers.update(DEFAULT_HTTP_HEADERS)    
+    resp = self._client.request(method, url, json=body, params=params, timeout=timeout,
+                                verify=MITM_CERT_PATH, headers=headers)
+    if resp.status_code // 100 != 2:
+      raise Non200ResponseException(resp)
+    return resp
+
+  def _call(self, method, path, params=None, body=None, headers=None):
+    """ Issues an HTTP request to signing service and handles failover for GET requests.
+    """
+    if self._config is None:
+      raise Exception('Cannot call unconfigured signing service')
+
+    timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1)
+    endpoint = self._config['TUF_SERVER']
+
+    with CloseForLongOperation(self._config):
+      # If the request isn't a read do not fail over.
+      if method != 'GET':
+        return self._request(method, endpoint, path, body, headers, params, timeout)
+
+      # The request is read-only and can failover.
+      all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', [])
+      return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {})
+                                      for endpoint in all_endpoints])
+
+
+@failover
+def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout):
+  """ This function auto-retries read-only requests until they return a 2xx status code. """
+  try:
+    return request_fn('GET', endpoint, path, body, headers, params, timeout)
+  except (requests.exceptions.RequestException, Non200ResponseException) as ex:
+    raise FailoverException(ex)

util/tufmetadata/test/test_tufmetadata.py

@@ -0,0 +1,71 @@
+import pytest
+import requests
+from mock import mock
+
+from flask import Flask
+
+from test import testconfig
+from util.tufmetadata import api 
+
+valid_response = {
+  'signed' : {
+    'type': 'Targets',
+    'delegations': {
+      'keys': {},
+      'roles': {},
+    },
+    'expires': '2020-03-30T18:55:26.594764859-04:00',
+    'targets': {
+      'latest': {
+        'hashes': {
+          'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
+        },
+        'length': 1500
+      }
+    },
+    'version': 2
+  },
+  'signatures': [
+    {
+      'method': 'ecdsa',
+      'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw=='
+    }
+  ]
+}
+
+
+@pytest.mark.parametrize('response_code,response_body,expected', [
+  (200, valid_response, valid_response['signed']['targets']),
+  (200, {'garbage': 'data'}, None)
+])
+def test_get_metadata(response_code, response_body, expected): 
+    app = Flask(__name__) 
+    app.config.from_object(testconfig.TestConfig())
+    client = mock.Mock()
+    request = mock.Mock(status_code=response_code)
+    request.json.return_value = response_body
+    client.request.return_value = request
+    tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
+    tags, _ = tuf_api.get_default_tags('quay', 'quay')
+    assert tags == expected 
+    
+@pytest.mark.parametrize('connection_error,response_code,exception', [
+  (True, 200, requests.exceptions.Timeout),
+  (True, 200, requests.exceptions.ConnectionError),
+  (False, 200, requests.exceptions.RequestException),
+  (False, 200, ValueError),
+  (True, 500, api.Non200ResponseException(mock.Mock(status_code=500))),
+  (False, 400, api.Non200ResponseException(mock.Mock(status_code=400))),
+  (False, 404, api.Non200ResponseException(mock.Mock(status_code=404))),
+  (False, 200, api.InvalidMetadataException)
+  
+])
+def test_get_metadata_exception(connection_error, response_code, exception): 
+    app = Flask(__name__) 
+    app.config.from_object(testconfig.TestConfig())
+    request = mock.Mock(status_code=response_code)
+    client = mock.Mock(request=request)
+    client.request.side_effect = exception
+    tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
+    tags, _ = tuf_api.get_default_tags('quay', 'quay')
+    assert tags == None