Merge pull request #2512 from ecordell/tufmetadata

Add tufmetadata endpoint

util/tufmetadata/api.py

@@ -0,0 +1,130 @@
+import logging
+from urlparse import urljoin
+
+import requests
+
+from data.database import CloseForLongOperation
+from util.failover import failover, FailoverException
+from util.security.instancekeys import InstanceKeys
+from util.security.registry_jwt import build_context_and_subject, generate_bearer_token, QUAY_TUF_ROOT
+
+
+DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
+MITM_CERT_PATH = '/conf/mitm.cert'
+TOKEN_VALIDITY_LIFETIME_S = 60 * 60  # 1 hour
+
+logger = logging.getLogger(__name__)
+
+
+class InvalidMetadataException(Exception):
+  """ Exception raised when the upstream API metadata that doesn't parse correctly. """
+  pass
+
+
+class Non200ResponseException(Exception):
+  """ Exception raised when the upstream API returns a non-200 HTTP status code. """
+  def __init__(self, response):
+    super(Non200ResponseException, self).__init__()
+    self.response = response
+
+
+class TUFMetadataAPI(object):
+  """ Helper class for talking to the TUF Metadata service (Apostille). """
+  def __init__(self, app, config, client=None):
+    self._app = app
+    self._instance_keys = InstanceKeys(app)
+    self._config = config
+    self._client = client or config['HTTPCLIENT']
+
+  def get_default_tags_with_expiration(self, namespace, repository, targets_file=None):
+    """ Gets the tag -> sha mappings in the 'targets/releases' delegation
+        Returns tags, their hashes, and their 
+    """
+    if not targets_file:
+      targets_file = 'targets/releases.json'
+    gun = "%s/%s" % (namespace, repository)
+    
+    try:
+      response = self._get(gun, targets_file)
+      signed = self._parse_signed(response.json())
+      targets = signed.get('targets')
+      expiration = signed.get('expires')
+    except requests.exceptions.Timeout:
+      logger.exception('Timeout when trying to get metadata for %s', gun)
+      return None, None 
+    except requests.exceptions.ConnectionError:
+      logger.exception('Connection error when trying to get metadata for %s', gun)
+      return None, None 
+    except (requests.exceptions.RequestException, ValueError):
+      logger.exception('Failed to get metadata for %s', gun)
+      return None, None 
+    except Non200ResponseException as ex:
+      logger.exception('Failed request for %s: %s', gun, str(ex))
+      return None, None 
+    except InvalidMetadataException as ex:
+      logger.exception('Failed to parse targets from metadata', str(ex))
+      return None, None 
+    
+    return targets, expiration
+  
+  def _parse_signed(self, json_response):
+    """ Attempts to parse the targets from a metadata response """
+    signed = json_response.get('signed')
+    if not signed:
+      raise InvalidMetadataException("Could not find `signed` in metadata: %s" % json_response)
+    return signed 
+  
+  def _auth_header(self, gun):
+    """ Generate a registry auth token for apostille"""
+    access = [{
+      'type': 'repository',
+      'name': '%s/%s' % (self._config['SERVER_HOSTNAME'], gun),
+      'actions': ['pull'],
+    }]
+    context, subject = build_context_and_subject(user=None, token=None, oauthtoken=None, tuf_root=QUAY_TUF_ROOT)
+    token = generate_bearer_token("quay", subject, context, access,
+                                  TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
+    return {'Authorization': 'Bearer %s' % token}
+    
+  def _get(self, gun, metadata_file):
+    return self._call('GET', '/v2/%s/_trust/tuf/%s' % (gun, metadata_file), headers=self._auth_header(gun))
+    
+  def _request(self, method, endpoint, path, body, headers, params, timeout):
+    """ Issues an HTTP request to the signing endpoint. """
+    url = urljoin(endpoint, path)
+    logger.debug('%sing signing URL %s', method.upper(), url)
+
+    headers.update(DEFAULT_HTTP_HEADERS)    
+    resp = self._client.request(method, url, json=body, params=params, timeout=timeout,
+                                verify=MITM_CERT_PATH, headers=headers)
+    if resp.status_code // 100 != 2:
+      raise Non200ResponseException(resp)
+    return resp
+
+  def _call(self, method, path, params=None, body=None, headers=None):
+    """ Issues an HTTP request to signing service and handles failover for GET requests.
+    """
+    if self._config is None:
+      raise Exception('Cannot call unconfigured signing service')
+
+    timeout = self._config.get('TUF_API_TIMEOUT_SECONDS', 1)
+    endpoint = self._config['TUF_SERVER']
+
+    with CloseForLongOperation(self._config):
+      # If the request isn't a read do not fail over.
+      if method != 'GET':
+        return self._request(method, endpoint, path, body, headers, params, timeout)
+
+      # The request is read-only and can failover.
+      all_endpoints = [endpoint] + self._config.get('TUF_READONLY_FAILOVER_ENDPOINTS', [])
+      return _failover_read_request(*[((self._request, endpoint, path, body, headers, params, timeout), {})
+                                      for endpoint in all_endpoints])
+
+
+@failover
+def _failover_read_request(request_fn, endpoint, path, body, headers, params, timeout):
+  """ This function auto-retries read-only requests until they return a 2xx status code. """
+  try:
+    return request_fn('GET', endpoint, path, body, headers, params, timeout)
+  except (requests.exceptions.RequestException, Non200ResponseException) as ex:
+    raise FailoverException(ex)

util/tufmetadata/test/test_tufmetadata.py

@@ -0,0 +1,72 @@
+import pytest
+import requests
+from mock import mock
+
+from flask import Flask
+
+from test import testconfig
+from util.tufmetadata import api 
+
+valid_response = {
+  'signed' : {
+    'type': 'Targets',
+    'delegations': {
+      'keys': {},
+      'roles': {},
+    },
+    'expires': '2020-03-30T18:55:26.594764859-04:00',
+    'targets': {
+      'latest': {
+        'hashes': {
+          'sha256': 'mLmxwTyUrqIRDaz8uaBapfrp3GPERfsDg2kiMujlteo='
+        },
+        'length': 1500
+      }
+    },
+    'version': 2
+  },
+  'signatures': [
+    {
+      'method': 'ecdsa',
+      'sig': 'yYnJGsYAYEL9PSwXisdG7JUEM1YK2IIKM147K3fWJthF4w+vl3xOm67r5ZNuDK6ss/Ff+x8yljZdT3sE/Hg5mw=='
+    }
+  ]
+}
+
+
+@pytest.mark.parametrize('response_code,response_body,expected', [
+  (200, valid_response, (valid_response['signed']['targets'], '2020-03-30T18:55:26.594764859-04:00')),
+  (200, {'garbage': 'data'}, (None, None))
+])
+def test_get_metadata(response_code, response_body, expected): 
+    app = Flask(__name__) 
+    app.config.from_object(testconfig.TestConfig())
+    client = mock.Mock()
+    request = mock.Mock(status_code=response_code)
+    request.json.return_value = response_body
+    client.request.return_value = request
+    tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
+    response = tuf_api.get_default_tags_with_expiration('quay', 'quay')
+    assert response == expected 
+    
+@pytest.mark.parametrize('connection_error,response_code,exception', [
+  (True, 200, requests.exceptions.Timeout),
+  (True, 200, requests.exceptions.ConnectionError),
+  (False, 200, requests.exceptions.RequestException),
+  (False, 200, ValueError),
+  (True, 500, api.Non200ResponseException(mock.Mock(status_code=500))),
+  (False, 400, api.Non200ResponseException(mock.Mock(status_code=400))),
+  (False, 404, api.Non200ResponseException(mock.Mock(status_code=404))),
+  (False, 200, api.InvalidMetadataException)
+  
+])
+def test_get_metadata_exception(connection_error, response_code, exception): 
+    app = Flask(__name__) 
+    app.config.from_object(testconfig.TestConfig())
+    request = mock.Mock(status_code=response_code)
+    client = mock.Mock(request=request)
+    client.request.side_effect = exception
+    tuf_api = api.TUFMetadataAPI(app, app.config, client=client)
+    tags, expiration = tuf_api.get_default_tags_with_expiration('quay', 'quay')
+    assert tags == None
+    assert expiration == None