Add a SecScanEndpoint class and move all the cert and config handling in there

app.py

@@ -35,6 +35,7 @@ from util.saas.metricqueue import MetricQueue
 from util.config.provider import get_config_provider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
+from util.secscan.secscanendpoint import SecurityScanEndpoint
 
 OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
 OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
@@ -147,6 +148,7 @@ image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf)
 dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
                                    reporter=MetricQueueReporter(metric_queue))
 notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
+secscan_endpoint = SecurityScanEndpoint(app, config_provider)
 
 database.configure(app.config)
 model.config.app_config = app.config

config.py

@@ -256,5 +256,6 @@ class DefaultConfig(object):
   SECURITY_SCANNER = {
     'ENDPOINT': 'http://192.168.99.100:6060',
     'ENGINE_VERSION_TARGET': 1,
-    'API_CALL_TIMEOUT': 10,
+    'API_VERSION': 'v1',
+    'API_TIMEOUT_SECONDS': 10,
   }

endpoints/api/__init__.py

@@ -423,5 +423,5 @@ import endpoints.api.tag
 import endpoints.api.team
 import endpoints.api.trigger
 import endpoints.api.user
-import endpoints.api.sec
+import endpoints.api.secscan
 

endpoints/api/secscan.py

@@ -2,11 +2,10 @@
 
 import logging
 import features
-import requests
 import json
+import requests
 
-from urlparse import urljoin
-from app import app
+from app import secscan_endpoint
 from data import model
 from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
                            RepositoryParamResource, resource, nickname, show_if, parse_args,
@@ -15,17 +14,11 @@ from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_pa
 
 logger = logging.getLogger(__name__)
 
+
 def _call_security_api(relative_url, *args, **kwargs):
   """ Issues an HTTP call to the sec API at the given relative URL. """
-  url = urljoin(app.config['SECURITY_SCANNER']['ENDPOINT'], relative_url % args)
-
-  client = app.config['HTTPCLIENT']
-  timeout = app.config['SECURITY_SCANNER'].get('API_CALL_TIMEOUT', 1)
-
-  logger.debug('Looking up sec information: %s', url)
-
   try:
-    response = client.get(url, params=kwargs, timeout=timeout)
+    response = secscan_endpoint.call_api(relative_url, *args, **kwargs)
   except requests.exceptions.Timeout:
     raise DownstreamIssue(payload=dict(message='API call timed out'))
   except requests.exceptions.ConnectionError:
@@ -40,8 +33,7 @@ def _call_security_api(relative_url, *args, **kwargs):
     raise DownstreamIssue(payload=dict(message='Non-json response from downstream service'))
 
   if response.status_code / 100 != 2:
-    logger.warning('Got %s status code to call %s: %s', response.status_code, url,
-                   response.text)
+    logger.warning('Got %s status code to call: %s', response.status_code, response.text)
     raise DownstreamIssue(payload=dict(message=response_data['Message']))
 
   return response_data
@@ -73,7 +65,7 @@ class RepositoryTagVulnerabilities(RepositoryParamResource):
         'security_indexed': False
       }
 
-    data = _call_security_api('/layers/%s/vulnerabilities', tag_image.docker_image_id,
+    data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id,
                               minimumPriority=args.minimumPriority)
 
     return {
@@ -102,7 +94,7 @@ class RepositoryImagePackages(RepositoryParamResource):
         'security_indexed': False
       }
 
-    data = _call_security_api('/layers/%s/packages', repo_image.docker_image_id)
+    data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id)
 
     return {
       'security_indexed': True,

initdb.py

@@ -95,10 +95,6 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
       for path_builder in paths:
         path = path_builder(new_image.storage.uuid)
         store.put_content('local_us', path, checksum)
-    
-    new_image.security_indexed = False
-    new_image.security_indexed_engine = maxsize
-    new_image.save()
 
     new_image.security_indexed = False
     new_image.security_indexed_engine = maxsize

test/test_api_security.py

@@ -50,7 +50,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana
                                      SuperUserOrganizationManagement, SuperUserOrganizationList,
                                      SuperUserAggregateLogs)
 
-from endpoints.api.sec import RepositoryImagePackages, RepositoryTagVulnerabilities
+from endpoints.api.secscan import RepositoryImagePackages, RepositoryTagVulnerabilities
 
 
 try:

test/test_api_usage.py

@@ -1549,8 +1549,8 @@ class TestDeleteRepository(ApiTestCase):
     model.build.create_repository_build(repository, delegate_token, {}, 'someid2', 'foobar2')
 
     # Create some notifications.
-    model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {})
-    model.notification.create_repo_notification(repository, 'build_queued', 'slack', {})
+    model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {}, {})
+    model.notification.create_repo_notification(repository, 'build_queued', 'slack', {}, {})
 
     # Create some logs.
     model.log.log_action('push_repo', ADMIN_ACCESS_USER, repository=repository)
@@ -1984,7 +1984,7 @@ class TestRepositoryNotifications(ApiTestCase):
     json = self.postJsonResponse(RepositoryNotificationList,
                                  params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
                                  data=dict(config={'url': 'http://example.com'}, event='repo_push',
-                                           method='webhook'),
+                                           method='webhook', eventConfig={}),
                                  expected_code=201)
 
     self.assertEquals('repo_push', json['event'])
@@ -2024,7 +2024,8 @@ class TestRepositoryNotifications(ApiTestCase):
     json = self.postJsonResponse(RepositoryNotificationList,
                                  params=dict(repository=ADMIN_ACCESS_USER + '/simple'),
                                  data=dict(config={'url': 'http://example.com'}, event='repo_push',
-                                           method='webhook', title='Some Notification'),
+                                           method='webhook', title='Some Notification',
+                                           eventConfig={}),
                                  expected_code=201)
 
     self.assertEquals('repo_push', json['event'])

util/secscan/secscanendpoint.py

@@ -0,0 +1,50 @@
+import features
+import logging
+import requests
+import json
+
+from urlparse import urljoin
+
+logger = logging.getLogger(__name__)
+
+class SecurityScanEndpoint(object):
+  """ Helper class for talking to the Security Scan service (Clair). """
+  def __init__(self, app, config_provider):
+    self.app = app
+    self.config_provider = config_provider
+
+    if not features.SECURITY_SCANNER:
+      return
+
+    self.security_config = app.config['SECURITY_SCANNER']
+
+    self.certificate = self._getfilepath('CA_CERTIFICATE_FILENAME') or False
+    self.public_key = self._getfilepath('PUBLIC_KEY_FILENAME')
+    self.private_key = self._getfilepath('PRIVATE_KEY_FILENAME')
+
+    if self.public_key and self.private_key:
+      self.keys = (self.public_key, self.private_key)
+    else:
+      self.keys = None
+
+  def _getfilepath(self, config_key):
+    security_config = self.security_config
+
+    if config_key in security_config:
+      with self.config_provider.get_volume_file(security_config[config_key]) as f:
+        return f.name
+
+    return None
+
+  def call_api(self, relative_url, *args, **kwargs):
+    """ Issues an HTTP call to the sec API at the given relative URL. """
+    security_config = self.security_config
+    api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
+    url = urljoin(api_url, relative_url % args)
+
+    client = self.app.config['HTTPCLIENT']
+    timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
+    logger.debug('Looking up sec information: %s', url)
+
+    return client.get(url, params=kwargs, timeout=timeout, cert=self.keys,
+                      verify=self.certificate)