Add vulnerabilities and packages API to Quay

Fixes #564

config.py

@@ -256,4 +256,5 @@ class DefaultConfig(object):
   SECURITY_SCANNER = {
     'ENDPOINT': 'http://192.168.99.100:6060',
     'ENGINE_VERSION_TARGET': 1,
+    'API_CALL_TIMEOUT': 10,
   }

endpoints/api/__init__.py

@@ -93,6 +93,11 @@ class NotFound(ApiException):
     ApiException.__init__(self, None, 404, 'Not Found', payload)
 
 
+class DownstreamIssue(ApiException):
+  def __init__(self, payload=None):
+    ApiException.__init__(self, None, 520, 'Downstream Issue', payload)
+
+
 @api_bp.app_errorhandler(ApiException)
 @crossdomain(origin='*', headers=['Authorization', 'Content-Type'])
 def handle_api_error(error):
@@ -418,4 +423,5 @@ import endpoints.api.tag
 import endpoints.api.team
 import endpoints.api.trigger
 import endpoints.api.user
+import endpoints.api.sec
 

endpoints/api/sec.py

@@ -0,0 +1,111 @@
+""" List and manage repository vulnerabilities and other sec information. """
+
+import logging
+import features
+import requests
+import json
+
+from urlparse import urljoin
+from app import app
+from data import model
+from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
+                           RepositoryParamResource, resource, nickname, show_if, parse_args,
+                           query_param)
+
+
+logger = logging.getLogger(__name__)
+
+def _call_security_api(relative_url, *args, **kwargs):
+  """ Issues an HTTP call to the sec API at the given relative URL. """
+  url = urljoin(app.config['SECURITY_SCANNER']['ENDPOINT'], relative_url % args)
+
+  client = app.config['HTTPCLIENT']
+  timeout = app.config['SECURITY_SCANNER'].get('API_CALL_TIMEOUT', 1)
+
+  logger.debug('Looking up sec information: %s', url)
+
+  try:
+    response = client.get(url, params=kwargs, timeout=timeout)
+  except requests.exceptions.Timeout:
+    raise DownstreamIssue(payload=dict(message='API call timed out'))
+  except requests.exceptions.ConnectionError:
+    raise DownstreamIssue(payload=dict(message='Could not connect to downstream service'))
+
+  if response.status_code == 404:
+    raise NotFound()
+
+  try:
+    response_data = json.loads(response.text)
+  except ValueError:
+    raise DownstreamIssue(payload=dict(message='Non-json response from downstream service'))
+
+  if response.status_code / 100 != 2:
+    logger.warning('Got %s status code to call %s: %s', response.status_code, url,
+                   response.text)
+    raise DownstreamIssue(payload=dict(message=response_data['Message']))
+
+  return response_data
+
+
+@show_if(features.SECURITY_SCANNER)
+@resource('/v1/repository/<repopath:repository>/tag/<tag>/vulnerabilities')
+@path_param('repository', 'The full path of the repository. e.g. namespace/name')
+@path_param('tag', 'The name of the tag')
+class RepositoryTagVulnerabilities(RepositoryParamResource):
+  """ Operations for managing the vulnerabilities in a repository tag. """
+
+  @require_repo_read
+  @nickname('getRepoTagVulnerabilities')
+  @parse_args
+  @query_param('minimumPriority', 'Minimum vulnerability priority', type=str,
+               default='Low')
+  def get(self, args, namespace, repository, tag):
+    """ Fetches the vulnerabilities (if any) for a repository tag. """
+    try:
+      tag_image = model.tag.get_tag_image(namespace, repository, tag)
+    except model.DataModelException:
+      raise NotFound()
+
+    if not tag_image.security_indexed:
+      logger.debug('Image %s for tag %s under repository %s/%s not security indexed',
+                   tag_image.docker_image_id, tag, namespace, repository)
+      return {
+        'security_indexed': False
+      }
+
+    data = _call_security_api('/layers/%s/vulnerabilities', tag_image.docker_image_id,
+                              minimumPriority=args.minimumPriority)
+
+    return {
+      'security_indexed': True,
+      'data': data,
+    }
+
+
+@show_if(features.SECURITY_SCANNER)
+@resource('/v1/repository/<repopath:repository>/image/<imageid>/packages')
+@path_param('repository', 'The full path of the repository. e.g. namespace/name')
+@path_param('imageid', 'The image ID')
+class RepositoryImagePackages(RepositoryParamResource):
+  """ Operations for listing the packages added/removed in an image. """
+
+  @require_repo_read
+  @nickname('getRepoImagePackages')
+  def get(self, namespace, repository, imageid):
+    """ Fetches the packages added/removed in the given repo image. """
+    repo_image = model.image.get_repo_image(namespace, repository, imageid)
+    if repo_image is None:
+      raise NotFound()
+
+    if not repo_image.security_indexed:
+      return {
+        'security_indexed': False
+      }
+
+    data = _call_security_api('/layers/%s/packages', repo_image.docker_image_id)
+
+    return {
+      'security_indexed': True,
+      'data': data,
+    }
+

test/test_api_security.py

@@ -50,6 +50,8 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
                                      SuperUserOrganizationManagement, SuperUserOrganizationList,
                                      SuperUserAggregateLogs)
 
+from endpoints.api.sec import RepositoryImagePackages, RepositoryTagVulnerabilities
+
 
 try:
   app.register_blueprint(api_bp, url_prefix='/api')
@@ -4210,18 +4212,54 @@ class TestOrganizationInvoiceField(ApiTestCase):
     ApiTestCase.setUp(self)
     self._set_url(OrganizationInvoiceField, orgname='buynlarge', field_uuid='1234')
 
-  def test_get_anonymous(self):
+  def test_delete_anonymous(self):
     self._run_test('DELETE', 403, None, None)
 
-  def test_get_freshuser(self):
+  def test_delete_freshuser(self):
     self._run_test('DELETE', 403, 'freshuser', None)
 
-  def test_get_reader(self):
+  def test_delete_reader(self):
     self._run_test('DELETE', 403, 'reader', None)
 
-  def test_get_devtable(self):
+  def test_delete_devtable(self):
     self._run_test('DELETE', 201, 'devtable', None)
 
 
+class TestRepositoryTagVulnerabilities(ApiTestCase):
+  def setUp(self):
+    ApiTestCase.setUp(self)
+    self._set_url(RepositoryTagVulnerabilities, repository='devtable/simple', tag='latest')
+
+  def test_get_anonymous(self):
+    self._run_test('GET', 401, None, None)
+
+  def test_get_freshuser(self):
+    self._run_test('GET', 403, 'freshuser', None)
+
+  def test_get_reader(self):
+    self._run_test('GET', 403, 'reader', None)
+
+  def test_get_devtable(self):
+    self._run_test('GET', 200, 'devtable', None)
+
+
+class TestRepositoryImagePackages(ApiTestCase):
+  def setUp(self):
+    ApiTestCase.setUp(self)
+    self._set_url(RepositoryImagePackages, repository='devtable/simple', imageid='fake')
+
+  def test_get_anonymous(self):
+    self._run_test('GET', 401, None, None)
+
+  def test_get_freshuser(self):
+    self._run_test('GET', 403, 'freshuser', None)
+
+  def test_get_reader(self):
+    self._run_test('GET', 403, 'reader', None)
+
+  def test_get_devtable(self):
+    self._run_test('GET', 404, 'devtable', None)
+
+
 if __name__ == '__main__':
   unittest.main()

test/testconfig.py

@@ -55,3 +55,10 @@ class TestConfig(DefaultConfig):
   FEATURE_GITHUB_BUILD = True
 
   CLOUDWATCH_NAMESPACE = None
+
+  FEATURE_SECURITY_SCANNER = True
+  SECURITY_SCANNER = {
+    'ENDPOINT': 'http://localhost/some/invalid/path',
+    'ENGINE_VERSION_TARGET': 1,
+    'API_CALL_TIMEOUT': 1
+  }