diff --git a/app.py b/app.py index 0fcda573c..01c4ae6ab 100644 --- a/app.py +++ b/app.py @@ -35,6 +35,7 @@ from util.saas.metricqueue import MetricQueue from util.config.provider import get_config_provider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager +from util.secscan.secscanendpoint import SecurityScanEndpoint OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' @@ -147,6 +148,7 @@ image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, reporter=MetricQueueReporter(metric_queue)) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) +secscan_endpoint = SecurityScanEndpoint(app, config_provider) database.configure(app.config) model.config.app_config = app.config diff --git a/config.py b/config.py index 98f7470ad..c1999cde4 100644 --- a/config.py +++ b/config.py @@ -256,5 +256,6 @@ class DefaultConfig(object): SECURITY_SCANNER = { 'ENDPOINT': 'http://192.168.99.100:6060', 'ENGINE_VERSION_TARGET': 1, - 'API_CALL_TIMEOUT': 10, + 'API_VERSION': 'v1', + 'API_TIMEOUT_SECONDS': 10, } diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 599899ca7..a83a7ac58 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -423,5 +423,5 @@ import endpoints.api.tag import endpoints.api.team import endpoints.api.trigger import endpoints.api.user -import endpoints.api.sec +import endpoints.api.secscan diff --git a/endpoints/api/sec.py b/endpoints/api/secscan.py similarity index 83% rename from endpoints/api/sec.py rename to endpoints/api/secscan.py index e4e571414..ab3f73051 100644 --- a/endpoints/api/sec.py +++ b/endpoints/api/secscan.py @@ -2,11 +2,10 @@ import logging import features -import requests import json +import requests -from urlparse import urljoin -from app import app +from app import secscan_endpoint from data import model from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_param, RepositoryParamResource, resource, nickname, show_if, parse_args, @@ -15,17 +14,11 @@ from endpoints.api import (require_repo_read, NotFound, DownstreamIssue, path_pa logger = logging.getLogger(__name__) + def _call_security_api(relative_url, *args, **kwargs): """ Issues an HTTP call to the sec API at the given relative URL. """ - url = urljoin(app.config['SECURITY_SCANNER']['ENDPOINT'], relative_url % args) - - client = app.config['HTTPCLIENT'] - timeout = app.config['SECURITY_SCANNER'].get('API_CALL_TIMEOUT', 1) - - logger.debug('Looking up sec information: %s', url) - try: - response = client.get(url, params=kwargs, timeout=timeout) + response = secscan_endpoint.call_api(relative_url, *args, **kwargs) except requests.exceptions.Timeout: raise DownstreamIssue(payload=dict(message='API call timed out')) except requests.exceptions.ConnectionError: @@ -40,8 +33,7 @@ def _call_security_api(relative_url, *args, **kwargs): raise DownstreamIssue(payload=dict(message='Non-json response from downstream service')) if response.status_code / 100 != 2: - logger.warning('Got %s status code to call %s: %s', response.status_code, url, - response.text) + logger.warning('Got %s status code to call: %s', response.status_code, response.text) raise DownstreamIssue(payload=dict(message=response_data['Message'])) return response_data @@ -73,7 +65,7 @@ class RepositoryTagVulnerabilities(RepositoryParamResource): 'security_indexed': False } - data = _call_security_api('/layers/%s/vulnerabilities', tag_image.docker_image_id, + data = _call_security_api('layers/%s/vulnerabilities', tag_image.docker_image_id, minimumPriority=args.minimumPriority) return { @@ -102,7 +94,7 @@ class RepositoryImagePackages(RepositoryParamResource): 'security_indexed': False } - data = _call_security_api('/layers/%s/packages', repo_image.docker_image_id) + data = _call_security_api('layers/%s/packages/diff', repo_image.docker_image_id) return { 'security_indexed': True, diff --git a/initdb.py b/initdb.py index 8b095b002..80c9fa952 100644 --- a/initdb.py +++ b/initdb.py @@ -95,10 +95,6 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map): for path_builder in paths: path = path_builder(new_image.storage.uuid) store.put_content('local_us', path, checksum) - - new_image.security_indexed = False - new_image.security_indexed_engine = maxsize - new_image.save() new_image.security_indexed = False new_image.security_indexed_engine = maxsize diff --git a/test/data/test.db b/test/data/test.db index 6b318ccd3..ab3922d08 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_security.py b/test/test_api_security.py index a58328887..11b33f71a 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -50,7 +50,7 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserOrganizationManagement, SuperUserOrganizationList, SuperUserAggregateLogs) -from endpoints.api.sec import RepositoryImagePackages, RepositoryTagVulnerabilities +from endpoints.api.secscan import RepositoryImagePackages, RepositoryTagVulnerabilities try: diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 0e23f6ae3..62224e657 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -1549,8 +1549,8 @@ class TestDeleteRepository(ApiTestCase): model.build.create_repository_build(repository, delegate_token, {}, 'someid2', 'foobar2') # Create some notifications. - model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {}) - model.notification.create_repo_notification(repository, 'build_queued', 'slack', {}) + model.notification.create_repo_notification(repository, 'repo_push', 'hipchat', {}, {}) + model.notification.create_repo_notification(repository, 'build_queued', 'slack', {}, {}) # Create some logs. model.log.log_action('push_repo', ADMIN_ACCESS_USER, repository=repository) @@ -1984,7 +1984,7 @@ class TestRepositoryNotifications(ApiTestCase): json = self.postJsonResponse(RepositoryNotificationList, params=dict(repository=ADMIN_ACCESS_USER + '/simple'), data=dict(config={'url': 'http://example.com'}, event='repo_push', - method='webhook'), + method='webhook', eventConfig={}), expected_code=201) self.assertEquals('repo_push', json['event']) @@ -2024,7 +2024,8 @@ class TestRepositoryNotifications(ApiTestCase): json = self.postJsonResponse(RepositoryNotificationList, params=dict(repository=ADMIN_ACCESS_USER + '/simple'), data=dict(config={'url': 'http://example.com'}, event='repo_push', - method='webhook', title='Some Notification'), + method='webhook', title='Some Notification', + eventConfig={}), expected_code=201) self.assertEquals('repo_push', json['event']) diff --git a/util/secscan/__init__.py b/util/secscan/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/util/secscan/secscanendpoint.py b/util/secscan/secscanendpoint.py new file mode 100644 index 000000000..7f759219b --- /dev/null +++ b/util/secscan/secscanendpoint.py @@ -0,0 +1,50 @@ +import features +import logging +import requests +import json + +from urlparse import urljoin + +logger = logging.getLogger(__name__) + +class SecurityScanEndpoint(object): + """ Helper class for talking to the Security Scan service (Clair). """ + def __init__(self, app, config_provider): + self.app = app + self.config_provider = config_provider + + if not features.SECURITY_SCANNER: + return + + self.security_config = app.config['SECURITY_SCANNER'] + + self.certificate = self._getfilepath('CA_CERTIFICATE_FILENAME') or False + self.public_key = self._getfilepath('PUBLIC_KEY_FILENAME') + self.private_key = self._getfilepath('PRIVATE_KEY_FILENAME') + + if self.public_key and self.private_key: + self.keys = (self.public_key, self.private_key) + else: + self.keys = None + + def _getfilepath(self, config_key): + security_config = self.security_config + + if config_key in security_config: + with self.config_provider.get_volume_file(security_config[config_key]) as f: + return f.name + + return None + + def call_api(self, relative_url, *args, **kwargs): + """ Issues an HTTP call to the sec API at the given relative URL. """ + security_config = self.security_config + api_url = urljoin(security_config['ENDPOINT'], '/' + security_config['API_VERSION']) + '/' + url = urljoin(api_url, relative_url % args) + + client = self.app.config['HTTPCLIENT'] + timeout = security_config.get('API_TIMEOUT_SECONDS', 1) + logger.debug('Looking up sec information: %s', url) + + return client.get(url, params=kwargs, timeout=timeout, cert=self.keys, + verify=self.certificate) \ No newline at end of file