From 8e2868737b095e56e5c7cd9c9147e6c9eec2ce85 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 10 Nov 2015 15:01:33 -0500 Subject: [PATCH] rename secscan_endpoint and move db close to API --- app.py | 4 ++-- endpoints/api/sec.py | 4 ++-- endpoints/api/secscan.py | 4 ++-- util/secscan/{secscanendpoint.py => api.py} | 26 +++++++++++++-------- workers/security_notification_worker.py | 21 +++++++---------- workers/securityworker.py | 4 ++-- 6 files changed, 33 insertions(+), 30 deletions(-) rename util/secscan/{secscanendpoint.py => api.py} (74%) diff --git a/app.py b/app.py index bbc8ecee1..964a79536 100644 --- a/app.py +++ b/app.py @@ -35,7 +35,7 @@ from util.saas.metricqueue import MetricQueue from util.config.provider import get_config_provider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager -from util.secscan.secscanendpoint import SecurityScanEndpoint +from util.secscan.api import SecurityScannerAPI OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' @@ -150,7 +150,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf) -secscan_endpoint = SecurityScanEndpoint(app, config_provider) +secscan_api = SecurityScannerAPI(app, config_provider) database.configure(app.config) model.config.app_config = app.config diff --git a/endpoints/api/sec.py b/endpoints/api/sec.py index d080d2fe1..4e77750f9 100644 --- a/endpoints/api/sec.py +++ b/endpoints/api/sec.py @@ -5,7 +5,7 @@ import features import json import requests -from app import secscan_endpoint +from app import secscan_api from data import model from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param, RepositoryParamResource, resource, nickname, show_if, parse_args, @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) def _call_security_api(relative_url, *args, **kwargs): """ Issues an HTTP call to the sec API at the given relative URL. """ try: - response = secscan_endpoint.call_api(relative_url, *args, **kwargs) + response = secscan_api.call(relative_url, *args, **kwargs) except requests.exceptions.Timeout: raise DownstreamIssue(payload=dict(message='API call timed out')) except requests.exceptions.ConnectionError: diff --git a/endpoints/api/secscan.py b/endpoints/api/secscan.py index 9a1773ccb..56fcea95d 100644 --- a/endpoints/api/secscan.py +++ b/endpoints/api/secscan.py @@ -5,7 +5,7 @@ import features import json import requests -from app import secscan_endpoint +from app import secscan_api from data import model from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param, RepositoryParamResource, resource, nickname, show_if, parse_args, @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) def _call_security_api(relative_url, *args, **kwargs): """ Issues an HTTP call to the sec API at the given relative URL. """ try: - response = secscan_endpoint.call_api(relative_url, body=None, *args, **kwargs) + response = secscan_api.call(relative_url, body=None, *args, **kwargs) except requests.exceptions.Timeout: raise DownstreamIssue(payload=dict(message='API call timed out')) except requests.exceptions.ConnectionError: diff --git a/util/secscan/secscanendpoint.py b/util/secscan/api.py similarity index 74% rename from util/secscan/secscanendpoint.py rename to util/secscan/api.py index 2cd24fab1..e03a19369 100644 --- a/util/secscan/secscanendpoint.py +++ b/util/secscan/api.py @@ -2,11 +2,13 @@ import features import logging import requests +from app import app +from database import CloseForLongOperation from urlparse import urljoin logger = logging.getLogger(__name__) -class SecurityScanEndpoint(object): +class SecurityScannerAPI(object): """ Helper class for talking to the Security Scan service (Clair). """ def __init__(self, app, config_provider): self.app = app @@ -41,7 +43,7 @@ class SecurityScanEndpoint(object): body = { 'LayersIDs': [layer_id] } - response = self.call_api('vulnerabilities/%s/affected-layers', body, cve_id) + response = self.call('vulnerabilities/%s/affected-layers', body, cve_id) except requests.exceptions.RequestException: logger.exception('Got exception when trying to call Clair endpoint') return False @@ -61,8 +63,11 @@ class SecurityScanEndpoint(object): return True - def call_api(self, relative_url, body=None, *args, **kwargs): - """ Issues an HTTP call to the sec API at the given relative URL. """ + def call(self, relative_url, body=None, *args, **kwargs): + """ Issues an HTTP call to the sec API at the given relative URL. + This function disconnects from the database while awaiting a response + from the API server. + """ security_config = self.security_config api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/' url = urljoin(api_url, relative_url % args) @@ -71,9 +76,10 @@ class SecurityScanEndpoint(object): timeout = security_config.get('API_TIMEOUT_SECONDS', 1) logger.debug('Looking up sec information: %s', url) - if body is not None: - return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self.keys, - verify=self.certificate) - else: - return client.get(url, params=kwargs, timeout=timeout, cert=self.keys, - verify=self.certificate) \ No newline at end of file + with CloseForLongOperation(app.config): + if body is not None: + return client.post(url, json=body, params=kwargs, timeout=timeout, cert=self.keys, + verify=self.certificate) + else: + return client.get(url, params=kwargs, timeout=timeout, cert=self.keys, + verify=self.certificate) diff --git a/workers/security_notification_worker.py b/workers/security_notification_worker.py index 8ec95f7d8..1679e80b6 100644 --- a/workers/security_notification_worker.py +++ b/workers/security_notification_worker.py @@ -6,10 +6,10 @@ from collections import defaultdict import features -from app import app, secscan_notification_queue, secscan_endpoint +from app import secscan_notification_queue, secscan_api from data import model from data.database import (Image, ImageStorage, ExternalNotificationEvent, - Repository, RepositoryNotification, RepositoryTag, CloseForLongOperation) + Repository, RepositoryNotification, RepositoryTag) from endpoints.notificationhelper import spawn_notification from workers.queueworker import QueueWorker @@ -42,11 +42,11 @@ class SecurityNotificationWorker(QueueWorker): Repository, Image, ImageStorage) # Additionally filter to tags only in repositories that have the event setup. - matching = (tags - .switch(RepositoryTag) - .join(Repository) - .join(RepositoryNotification) - .where(RepositoryNotification.event == event)) + matching = list(tags + .switch(RepositoryTag) + .join(Repository) + .join(RepositoryNotification) + .where(RepositoryNotification.event == event)) check_map = {} for tag in matching: @@ -55,9 +55,8 @@ class SecurityNotificationWorker(QueueWorker): logger.debug('Checking if layer %s is vulnerable to %s', tag_layer_id, cve_id) if not tag_layer_id in check_map: - with CloseForLongOperation(app.config): - is_vulerable = secscan_endpoint.check_layer_vulnerable(tag_layer_id, cve_id) - check_map[tag_layer_id] = is_vulerable + is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id) + check_map[tag_layer_id] = is_vulerable logger.debug('Result of layer %s is vulnerable to %s check: %s', tag_layer_id, cve_id, check_map[tag_layer_id]) @@ -80,8 +79,6 @@ class SecurityNotificationWorker(QueueWorker): }, } - # TODO(jschorr): only add this notification if the repository's event(s) defined meet - # the priority minimum. spawn_notification(repository_map[repository_id], 'vulnerability_found', event_data) diff --git a/workers/securityworker.py b/workers/securityworker.py index 65d71a8fe..26d360754 100644 --- a/workers/securityworker.py +++ b/workers/securityworker.py @@ -12,7 +12,7 @@ from endpoints.notificationhelper import spawn_notification from collections import defaultdict from sys import exc_info from peewee import JOIN_LEFT_OUTER -from app import app, storage, OVERRIDE_CONFIG_DIRECTORY, secscan_endpoint +from app import app, storage, OVERRIDE_CONFIG_DIRECTORY, secscan_api from workers.worker import Worker from data.database import (Image, ImageStorage, ImageStorageLocation, ImageStoragePlacement, db_random_func, UseThenDisconnect, RepositoryTag, Repository, @@ -256,7 +256,7 @@ class SecurityWorker(Worker): # callback code, etc. try: logger.debug('Loading vulnerabilities for layer %s', img['image_id']) - response = secscan_endpoint.call_api('layers/%s/vulnerabilities', request['ID']) + response = secscan_api.call('layers/%s/vulnerabilities', request['ID']) except requests.exceptions.Timeout: logger.debug('Timeout when calling Sec') continue