rename secscan_endpoint and move db close to API

app.py

@@ -35,7 +35,7 @@ from util.saas.metricqueue import MetricQueue
 from util.config.provider import get_config_provider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
-from util.secscan.secscanendpoint import SecurityScanEndpoint
+from util.secscan.api import SecurityScannerAPI
 
 OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
 OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
@@ -150,7 +150,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf
 notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
 secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf)
 
-secscan_endpoint = SecurityScanEndpoint(app, config_provider)
+secscan_api = SecurityScannerAPI(app, config_provider)
 
 database.configure(app.config)
 model.config.app_config = app.config

endpoints/api/sec.py

@@ -5,7 +5,7 @@ import features
 import json
 import requests
 
-from app import secscan_endpoint
+from app import secscan_api
 from data import model
 from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
                            RepositoryParamResource, resource, nickname, show_if, parse_args,
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 def _call_security_api(relative_url, *args, **kwargs):
   """ Issues an HTTP call to the sec API at the given relative URL. """
   try:
-    response = secscan_endpoint.call_api(relative_url, *args, **kwargs)
+    response = secscan_api.call(relative_url, *args, **kwargs)
   except requests.exceptions.Timeout:
     raise DownstreamIssue(payload=dict(message='API call timed out'))
   except requests.exceptions.ConnectionError:

endpoints/api/secscan.py

@@ -5,7 +5,7 @@ import features
 import json
 import requests
 
-from app import secscan_endpoint
+from app import secscan_api
 from data import model
 from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param,
                            RepositoryParamResource, resource, nickname, show_if, parse_args,
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
 def _call_security_api(relative_url, *args, **kwargs):
   """ Issues an HTTP call to the sec API at the given relative URL. """
   try:
-    response = secscan_endpoint.call_api(relative_url, body=None, *args, **kwargs)
+    response = secscan_api.call(relative_url, body=None, *args, **kwargs)
   except requests.exceptions.Timeout:
     raise DownstreamIssue(payload=dict(message='API call timed out'))
   except requests.exceptions.ConnectionError:

util/secscan/api.py

@@ -2,11 +2,13 @@ import features
 import logging
 import requests
 
+from app import app
+from database import CloseForLongOperation
 from urlparse import urljoin
 
 logger = logging.getLogger(__name__)
 
-class SecurityScanEndpoint(object):
+class SecurityScannerAPI(object):
   """ Helper class for talking to the Security Scan service (Clair). """
   def __init__(self, app, config_provider):
     self.app = app
@@ -41,7 +43,7 @@ class SecurityScanEndpoint(object):
       body = {
         'LayersIDs': [layer_id]
       }
-      response = self.call_api('vulnerabilities/%s/affected-layers', body, cve_id)
+      response = self.call('vulnerabilities/%s/affected-layers', body, cve_id)
     except requests.exceptions.RequestException:
       logger.exception('Got exception when trying to call Clair endpoint')
       return False
@@ -61,8 +63,11 @@ class SecurityScanEndpoint(object):
 
     return True
 
-  def call_api(self, relative_url, body=None, *args, **kwargs):
-    """ Issues an HTTP call to the sec API at the given relative URL. """
+  def call(self, relative_url, body=None, *args, **kwargs):
+    """ Issues an HTTP call to the sec API at the given relative URL.
+        This function disconnects from the database while awaiting a response
+        from the API server.
+    """
     security_config = self.security_config
     api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/'
     url = urljoin(api_url, relative_url % args)
@@ -71,9 +76,10 @@ class SecurityScanEndpoint(object):
     timeout = security_config.get('API_TIMEOUT_SECONDS', 1)
     logger.debug('Looking up sec information: %s', url)
 
-    if body is not None:
-      return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self.keys,
-                         verify=self.certificate)
-    else:
-      return client.get(url, params=kwargs, timeout=timeout, cert=self.keys,
-                        verify=self.certificate)
+    with CloseForLongOperation(app.config):
+      if body is not None:
+        return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self.keys,
+                           verify=self.certificate)
+      else:
+        return client.get(url, params=kwargs, timeout=timeout, cert=self.keys,
+                          verify=self.certificate)

workers/security_notification_worker.py

@@ -6,10 +6,10 @@ from collections import defaultdict
 
 import features
 
-from app import app, secscan_notification_queue, secscan_endpoint
+from app import secscan_notification_queue, secscan_api
 from data import model
 from data.database import (Image, ImageStorage, ExternalNotificationEvent,
-                           Repository, RepositoryNotification, RepositoryTag, CloseForLongOperation)
+                           Repository, RepositoryNotification, RepositoryTag)
 from endpoints.notificationhelper import spawn_notification
 from workers.queueworker import QueueWorker
 
@@ -42,11 +42,11 @@ class SecurityNotificationWorker(QueueWorker):
                                          Repository, Image, ImageStorage)
 
       # Additionally filter to tags only in repositories that have the event setup.
-      matching = (tags
-                  .switch(RepositoryTag)
-                  .join(Repository)
-                  .join(RepositoryNotification)
-                  .where(RepositoryNotification.event == event))
+      matching = list(tags
+                      .switch(RepositoryTag)
+                      .join(Repository)
+                      .join(RepositoryNotification)
+                      .where(RepositoryNotification.event == event))
 
       check_map = {}
       for tag in matching:
@@ -55,9 +55,8 @@ class SecurityNotificationWorker(QueueWorker):
         logger.debug('Checking if layer %s is vulnerable to %s', tag_layer_id, cve_id)
 
         if not tag_layer_id in check_map:
-          with CloseForLongOperation(app.config):
-            is_vulerable = secscan_endpoint.check_layer_vulnerable(tag_layer_id, cve_id)
-            check_map[tag_layer_id] = is_vulerable
+          is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id)
+          check_map[tag_layer_id] = is_vulerable
 
         logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id,
                      check_map[tag_layer_id])
@@ -80,8 +79,6 @@ class SecurityNotificationWorker(QueueWorker):
         },
       }
 
-      # TODO(jschorr): only add this notification if the repository's event(s) defined meet
-      # the priority minimum.
       spawn_notification(repository_map[repository_id], 'vulnerability_found', event_data)
 
 

workers/securityworker.py

@@ -12,7 +12,7 @@ from endpoints.notificationhelper import spawn_notification
 from collections import defaultdict
 from sys import exc_info
 from peewee import JOIN_LEFT_OUTER
-from app import app, storage, OVERRIDE_CONFIG_DIRECTORY, secscan_endpoint
+from app import app, storage, OVERRIDE_CONFIG_DIRECTORY, secscan_api
 from workers.worker import Worker
 from data.database import (Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement,
                            db_random_func, UseThenDisconnect, RepositoryTag, Repository,
@@ -256,7 +256,7 @@ class SecurityWorker(Worker):
           # callback code, etc.
           try:
             logger.debug('Loading vulnerabilities for layer %s', img['image_id'])
-            response = secscan_endpoint.call_api('layers/%s/vulnerabilities', request['ID'])
+            response = secscan_api.call('layers/%s/vulnerabilities', request['ID'])
           except requests.exceptions.Timeout:
             logger.debug('Timeout when calling Sec')
             continue