From 2cbdecb043426d8b92f4687a5c0566143743ebee Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 2 May 2016 15:29:31 -0400 Subject: [PATCH] Implement setup tool support for Clair Fixes #1387 --- app.py | 2 +- boot.py | 39 ++--- conf/jwtproxy_conf.yaml.jnj | 2 +- config.py | 31 +++- endpoints/api/superuser.py | 1 + static/css/core-ui.css | 19 +++ .../ui/request-service-key-dialog.css | 11 ++ .../config/config-service-key-field.html | 29 ++++ .../directives/config/config-setup-tool.html | 47 ++++++ .../request-service-key-dialog.html | 134 ++++++++++++++++++ static/js/core-config-setup.js | 85 +++++++++++ .../ui/request-service-key-dialog.js | 120 ++++++++++++++++ static/js/pages/setup.js | 5 +- static/js/pages/superuser.js | 2 +- static/partials/setup.html | 2 +- test/test_api_usage.py | 2 +- test/test_secscan.py | 2 +- test/testconfig.py | 10 +- util/config/configutil.py | 20 ++- util/config/validator.py | 25 +++- util/secscan/analyzer.py | 5 +- util/secscan/api.py | 63 ++++---- util/secscan/validator.py | 44 +----- 23 files changed, 584 insertions(+), 116 deletions(-) create mode 100644 static/css/directives/ui/request-service-key-dialog.css create mode 100644 static/directives/config/config-service-key-field.html create mode 100644 static/directives/request-service-key-dialog.html create mode 100644 static/js/directives/ui/request-service-key-dialog.js diff --git a/app.py b/app.py index 42bcaf514..970dfd95c 100644 --- a/app.py +++ b/app.py @@ -194,7 +194,7 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf reporter=MetricQueueReporter(metric_queue)) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf) -secscan_api = SecurityScannerAPI(app.config, config_provider, storage) +secscan_api = SecurityScannerAPI(app.config, storage) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) diff --git a/boot.py b/boot.py index a195b3ef6..a1990d253 100644 --- a/boot.py +++ b/boot.py @@ -5,8 +5,9 @@ from urlparse import urlunparse from jinja2 import Template from cachetools import lru_cache -import release +import release +import os.path from app import app from data.model.release import set_region_release @@ -37,49 +38,49 @@ def get_audience(): return urlunparse((scheme, hostname + ':' + port, '', '', '', '')) -def create_quay_service_key(): +def setup_jwt_proxy(): """ - Creates a service key for quay to use in the jwtproxy + Creates a service key for quay to use in the jwtproxy and generates the JWT proxy configuration. """ + if os.path.exists('conf/jwtproxy_conf.yaml'): + # Proxy is already setup. + return + + # Generate the key for this Quay instance to use. minutes_until_expiration = app.config.get('QUAY_SERVICE_KEY_EXPIRATION', 120) expiration = datetime.now() + timedelta(minutes=minutes_until_expiration) - quay_key, key_id = generate_key('quay', get_audience(), expiration_date=expiration) + quay_key, quay_key_id = generate_key('quay', get_audience(), expiration_date=expiration) - with open('/conf/quay.kid', mode='w') as f: + with open('conf/quay.kid', mode='w') as f: f.truncate(0) - f.write(key_id) + f.write(quay_key_id) - with open('/conf/quay.pem', mode='w') as f: + with open('conf/quay.pem', mode='w') as f: f.truncate(0) f.write(quay_key.exportKey()) - return key_id - - -def create_jwtproxy_conf(quay_key_id): - """ - Generates the jwtproxy conf from the jinja template - """ + # Generate the JWT proxy configuration. audience = get_audience() registry = audience + '/keys' + security_issuer = app.config.get('SECURITY_SCANNER_ISSUER_NAME', 'security_scanner') - with open("/conf/jwtproxy_conf.yaml.jnj") as f: + with open("conf/jwtproxy_conf.yaml.jnj") as f: template = Template(f.read()) rendered = template.render( audience=audience, registry=registry, - key_id=quay_key_id + key_id=quay_key_id, + security_issuer=security_issuer, ) - with open('/conf/jwtproxy_conf.yaml', 'w') as f: + with open('conf/jwtproxy_conf.yaml', 'w') as f: f.write(rendered) def main(): if app.config.get('SETUP_COMPLETE', False): sync_database_with_config(app.config) - quay_key_id = create_quay_service_key() - create_jwtproxy_conf(quay_key_id) + setup_jwt_proxy() # Record deploy if release.REGION and release.GIT_HEAD: diff --git a/conf/jwtproxy_conf.yaml.jnj b/conf/jwtproxy_conf.yaml.jnj index f657bf730..05f162400 100644 --- a/conf/jwtproxy_conf.yaml.jnj +++ b/conf/jwtproxy_conf.yaml.jnj @@ -23,5 +23,5 @@ jwtproxy: key_server: type: keyregistry options: - issuer: clair + issuer: {{ security_issuer }} registry: {{ registry }} diff --git a/config.py b/config.py index e61dab376..ba11fc48c 100644 --- a/config.py +++ b/config.py @@ -282,18 +282,33 @@ class DefaultConfig(object): # Security scanner FEATURE_SECURITY_SCANNER = False FEATURE_SECURITY_NOTIFICATIONS = False - SECURITY_SCANNER = { - 'ENDPOINT': 'http://192.168.99.101:6060', - 'ENGINE_VERSION_TARGET': 2, - 'API_VERSION': 'v1', - 'API_TIMEOUT_SECONDS': 10, - 'API_TIMEOUT_POST_SECONDS': 480, - } + + # The endpoint for the security scanner. + SECURITY_SCANNER_ENDPOINT = 'http://192.168.99.101:6060' + + # If specified, the endpoint to be used for all POST calls to the security scanner. + SECURITY_SCANNER_ENDPOINT_BATCH = None + + # The indexing engine version running inside the security scanner. + SECURITY_SCANNER_ENGINE_VERSION_TARGET = 2 + + # The version of the API to use for the security scanner. + SECURITY_SCANNER_API_VERSION = 'v1' + + # API call timeout for the security scanner. + SECURITY_SCANNER_API_TIMEOUT_SECONDS = 10 + + # POST call timeout for the security scanner. + SECURITY_SCANNER_API_TIMEOUT_POST_SECONDS = 480 + + # The issuer name for the security scanner. + SECURITY_SCANNER_ISSUER_NAME = 'security_scanner' # JWTProxy Settings # The address (sans schema) to proxy outgoing requests through the jwtproxy # to be signed JWTPROXY_SIGNER = 'localhost:8080' + # The audience that jwtproxy should verify on incoming requests # If None, will be calculated off of the SERVER_HOSTNAME (default) JWTPROXY_AUDIENCE = None @@ -322,7 +337,9 @@ class DefaultConfig(object): # The location of the private key generated for this instance INSTANCE_SERVICE_KEY_LOCATION = 'conf/quay.pem' + # This instance's service key expiration in minutes INSTANCE_SERVICE_KEY_EXPIRATION = 120 + # Number of minutes between expiration refresh in minutes INSTANCE_SERVICE_KEY_REFRESH = 60 diff --git a/endpoints/api/superuser.py b/endpoints/api/superuser.py index fc7ef8c8e..91edbb2f2 100644 --- a/endpoints/api/superuser.py +++ b/endpoints/api/superuser.py @@ -599,6 +599,7 @@ class SuperUserServiceKeyManagement(ApiResource): return jsonify({ 'kid': key.kid, 'name': body.get('name', ''), + 'service': body['service'], 'public_key': private_key.publickey().exportKey('PEM'), 'private_key': private_key.exportKey('PEM'), }) diff --git a/static/css/core-ui.css b/static/css/core-ui.css index 601b79981..70f69384e 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -545,6 +545,18 @@ a:focus { margin-left: 10px; } +.config-service-key-field-element { + position: relative; +} + +.config-service-key-field-element .co-modify-link { + margin-left: 10px; +} + +.config-service-key-field-element .fa-check { + margin-right: 4px; +} + .co-checkbox { position: relative; } @@ -1457,4 +1469,11 @@ a:focus { padding-bottom: 10px; } +.co-option-table .help-text { + margin-top: 4px; + margin-bottom: 10px; + font-size: 14px; + color: #aaa; +} + diff --git a/static/css/directives/ui/request-service-key-dialog.css b/static/css/directives/ui/request-service-key-dialog.css new file mode 100644 index 000000000..0ecfba910 --- /dev/null +++ b/static/css/directives/ui/request-service-key-dialog.css @@ -0,0 +1,11 @@ +.request-service-key-dialog-element .co-option-table { + margin-top: 20px; +} + +.request-service-key-dialog-element .key-display { + margin-top: 10px; + font-size: 12px; + font-family: Menlo,Monaco,Consolas,"Courier New",monospace; + background: white; + min-height: 500px; +} diff --git a/static/directives/config/config-service-key-field.html b/static/directives/config/config-service-key-field.html new file mode 100644 index 000000000..52b7c1187 --- /dev/null +++ b/static/directives/config/config-service-key-field.html @@ -0,0 +1,29 @@ +
+ +
+ + +
+ Could not load service keys +
+ + +
+
+ + Valid key for service {{ serviceName }} exists +
+
+ No valid key found for service {{ serviceName }} + Create Key +
+
+ + + + +
+
diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index b7386782e..2b758e252 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -286,6 +286,53 @@ + +
+
+ Security Scanner +
+
+
+

If enabled, all images pushed to Quay will be scanned via the external security scanning service, with vulnerability information available in the UI and API, as well + as async notification support. +

+
+ +
+ + +
+ +
+ A scanner compliant with the Quay Security Scanning API must be running to use this feature. Documentation on running Clair can be found at Running Clair Security Scanner. +
+ + + + + + + + + + +
Security Scanner Endpoint: + +
+ The HTTP URL at which the security scanner is running. +
+
Authentication Key: + +
+ The security scanning service requires an authorized service key to speak to Quay. Once setup, the key + can be managed in the Service Keys panel under the Super User Admin Panel. +
+
+
+
+
diff --git a/static/directives/request-service-key-dialog.html b/static/directives/request-service-key-dialog.html new file mode 100644 index 000000000..40ab659f4 --- /dev/null +++ b/static/directives/request-service-key-dialog.html @@ -0,0 +1,134 @@ +
+ +
+
+
+
+ +

Create key for service {{ requestKeyInfo.service }}

+
+ +
+
+
+ +
+ +
+ + + + + + + + + +
+ +
Recommended for {{ requestKeyInfo.service }} installations where the single instance is setup now.
+
+ +
Recommended for {{ requestKeyInfo.service }} installations where the instances are dynamically started.
+
+
+ + +
+
+ Please start the {{ requestKeyInfo.service }} service now, configured for autogenerated private key. The key approval process will continue automatically once the service connects to Quay. +
+
+ Waiting for service to connect +
+
+
+
+
+ + +
+ A key for service {{ requestKeyInfo.service }} has been automatically generated, approved and saved in the service's keystore. +
+ + +
+
+ + + + + + + + + + + + + +
+ + + A friendly name for the key for later reference. + +
+ + + The date and time that the key expires. If left blank, the key will never expire. + +
+
+ + Optional notes for additional human-readable information about why the key was created. + +
+
+
+ + +
+
+ The following key has been generated for service {{ requestKeyInfo.service }}. +

+ Please copy the key's ID and copy/download the key's private contents and place it in the directory with the service's configuration. +

+ Once this dialog is closed this private key will not be accessible anywhere else! +
+ + +
+ + + +
+
+
+ + + + + + + + + + +
+
+
+
+
+
\ No newline at end of file diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index bd1fa7620..83d9ce78a 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -61,6 +61,10 @@ angular.module("core-config-setup", ['angularFileUpload']) {'id': 'gitlab-trigger', 'title': 'GitLab Build Triggers', 'condition': function(config) { return config.FEATURE_GITLAB_BUILD; + }}, + + {'id': 'security-scanner', 'title': 'Quay Security Scanner', 'condition': function(config) { + return config.FEATURE_SECURITY_SCANNER; }} ]; @@ -1029,6 +1033,87 @@ angular.module("core-config-setup", ['angularFileUpload']) return directiveDefinitionObject; }) + .directive('configServiceKeyField', function (ApiService) { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/config/config-service-key-field.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'serviceName': '@serviceName', + }, + controller: function($scope, $element) { + $scope.foundKeys = []; + $scope.loading = false; + $scope.loadError = false; + $scope.hasValidKey = false; + $scope.hasValidKeyStr = null; + + $scope.updateKeys = function() { + $scope.foundKeys = []; + $scope.loading = true; + + ApiService.listServiceKeys().then(function(resp) { + $scope.loading = false; + $scope.loadError = false; + + resp['keys'].forEach(function(key) { + if (key['service'] == $scope.serviceName) { + $scope.foundKeys.push(key); + } + }); + + $scope.hasValidKey = checkValidKey($scope.foundKeys); + $scope.hasValidKeyStr = $scope.hasValidKey ? 'true' : ''; + }, function() { + $scope.loading = false; + $scope.loadError = true; + }); + }; + + // Perform initial loading of the keys. + $scope.updateKeys(); + + $scope.isKeyExpired = function(key) { + if (key.expiration_date != null) { + var expiration_date = moment(key.expiration_date); + return moment().isAfter(expiration_date); + } + return false; + }; + + $scope.showRequestServiceKey = function() { + $scope.requestKeyInfo = { + 'service': $scope.serviceName + }; + }; + + $scope.handleKeyCreated = function() { + $scope.updateKeys(); + }; + + var checkValidKey = function(keys) { + for (var i = 0; i < keys.length; ++i) { + var key = keys[i]; + if (!key.approval) { + continue; + } + + if ($scope.isKeyExpired(key)) { + continue; + } + + return true; + } + + return false; + }; + } + }; + return directiveDefinitionObject; + }) + .directive('configStringField', function () { var directiveDefinitionObject = { priority: 0, diff --git a/static/js/directives/ui/request-service-key-dialog.js b/static/js/directives/ui/request-service-key-dialog.js new file mode 100644 index 000000000..51b0ee98d --- /dev/null +++ b/static/js/directives/ui/request-service-key-dialog.js @@ -0,0 +1,120 @@ +/** + * An element which displays a dialog for requesting or creating a service key. + */ +angular.module('quay').directive('requestServiceKeyDialog', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/request-service-key-dialog.html', + replace: false, + transclude: true, + restrict: 'C', + scope: { + 'requestKeyInfo': '=requestKeyInfo', + 'keyCreated': '&keyCreated' + }, + controller: function($scope, $element, $timeout, ApiService) { + var handleNewKey = function(key) { + var data = { + 'notes': 'Approved during setup of service ' + key.service + }; + + var params = { + 'kid': key.kid + }; + + ApiService.approveServiceKey(data, params).then(function(resp) { + $scope.keyCreated({'key': key}); + $scope.step = 2; + }, ApiService.errorDisplay('Could not approve service key')); + }; + + var checkKeys = function() { + var isShown = ($element.find('.modal').data('bs.modal') || {}).isShown; + if (!isShown) { + return; + } + + // TODO: filter by service. + ApiService.listServiceKeys().then(function(resp) { + var keys = resp['keys']; + for (var i = 0; i < keys.length; ++i) { + var key = keys[i]; + if (key.service == $scope.requestKeyInfo.service && !key.approval && key.rotation_duration) { + handleNewKey(key); + return; + } + } + + $timeout(checkKeys, 1000); + }, ApiService.errorDisplay('Could not list service keys')); + }; + + $scope.show = function() { + $scope.working = false; + $scope.step = 0; + $scope.requestKind = null; + $scope.preshared = { + 'name': $scope.requestKeyInfo.service + ' Service Key', + 'notes': 'Created during setup for service `' + $scope.requestKeyInfo.service + '`' + }; + + $element.find('.modal').modal({}); + }; + + $scope.hide = function() { + $scope.loading = false; + $element.find('.modal').modal('hide'); + }; + + $scope.showGenerate = function() { + $scope.step = 1; + }; + + $scope.startApproval = function() { + $scope.step = 1; + checkKeys(); + }; + + $scope.isDownloadSupported = function() { + var isSafari = /^((?!chrome).)*safari/i.test(navigator.userAgent); + if (isSafari) { + // Doesn't work properly in Safari, sadly. + return false; + } + + try { return !!new Blob(); } catch(e) {} + return false; + }; + + $scope.downloadPrivateKey = function(key) { + var blob = new Blob([key.private_key]); + saveAs(blob, key.service + '.pem'); + }; + + $scope.createPresharedKey = function() { + $scope.working = true; + + var data = { + 'name': $scope.preshared.name, + 'service': $scope.requestKeyInfo.service, + 'expiration': $scope.preshared.expiration || null, + 'notes': $scope.preshared.notes + }; + + ApiService.createServiceKey(data).then(function(resp) { + $scope.working = false; + $scope.step = 2; + $scope.createdKey = resp; + $scope.keyCreated({'key': resp}); + }, ApiService.errorDisplay('Could not create service key')); + }; + + $scope.$watch('requestKeyInfo', function(info) { + if (info && info.service) { + $scope.show(); + } + }); + } + }; + return directiveDefinitionObject; +}); \ No newline at end of file diff --git a/static/js/pages/setup.js b/static/js/pages/setup.js index bb7c50092..036adbe31 100644 --- a/static/js/pages/setup.js +++ b/static/js/pages/setup.js @@ -1,13 +1,12 @@ (function() { /** - * The Setup page provides a nice GUI walkthrough experience for setting up the Enterprise - * Registry. + * The Setup page provides a nice GUI walkthrough experience for setting up Quay Enterprise. */ angular.module('quayPages').config(['pages', function(pages) { pages.create('setup', 'setup.html', SetupCtrl, { 'newLayout': true, - 'title': 'Enterprise Registry Setup' + 'title': 'Quay Enterprise Setup' }) }]); diff --git a/static/js/pages/superuser.js b/static/js/pages/superuser.js index 933c384ac..d97528398 100644 --- a/static/js/pages/superuser.js +++ b/static/js/pages/superuser.js @@ -1,6 +1,6 @@ (function() { /** - * The superuser admin page provides a new management UI for the Enterprise Registry. + * The superuser admin page provides a new management UI for Quay Enterprise. */ angular.module('quayPages').config(['pages', function(pages) { pages.create('superuser', 'super-user.html', SuperuserCtrl, diff --git a/static/partials/setup.html b/static/partials/setup.html index 23dca48c6..e28837b6e 100644 --- a/static/partials/setup.html +++ b/static/partials/setup.html @@ -3,7 +3,7 @@
- Enterprise Registry Setup + Quay Enterprise Setup
diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 0579cbec0..c6a75491c 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -3546,7 +3546,7 @@ class TestRepositoryImageSecurity(ApiTestCase): # Mark the layer as indexed. layer.security_indexed = True - layer.security_indexed_engine = app.config['SECURITY_SCANNER']['ENGINE_VERSION_TARGET'] + layer.security_indexed_engine = app.config['SECURITY_SCANNER_ENGINE_VERSION_TARGET'] layer.save() # Grab the security info again. diff --git a/test/test_secscan.py b/test/test_secscan.py index 8311d7792..219e262f7 100644 --- a/test/test_secscan.py +++ b/test/test_secscan.py @@ -122,7 +122,7 @@ class TestSecurityScanner(unittest.TestCase): self.ctx = app.test_request_context() self.ctx.__enter__() - self.api = SecurityScannerAPI(app.config, config_provider, storage) + self.api = SecurityScannerAPI(app.config, storage) def tearDown(self): storage.put_content(['local_us'], 'supports_direct_download', 'false') diff --git a/test/testconfig.py b/test/testconfig.py index b98da7016..2470e39b1 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -61,9 +61,7 @@ class TestConfig(DefaultConfig): FEATURE_SECURITY_SCANNER = True FEATURE_SECURITY_NOTIFICATIONS = True - SECURITY_SCANNER = { - 'ENDPOINT': 'http://mockclairservice/', - 'API_VERSION': 'v1', - 'ENGINE_VERSION_TARGET': 1, - 'API_TIMEOUT_SECONDS': 1 - } + SECURITY_SCANNER_ENDPOINT = 'http://mockclairservice/' + SECURITY_SCANNER_API_VERSION = 'v1' + SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1 + SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1 diff --git a/util/config/configutil.py b/util/config/configutil.py index 80c55f482..969de2a52 100644 --- a/util/config/configutil.py +++ b/util/config/configutil.py @@ -30,10 +30,20 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname): 'signing-public.gpg') config_obj['SIGNING_ENGINE'] = config_obj.get('SIGNING_ENGINE', 'gpg2') + # Default security scanner config. + config_obj['FEATURE_SECURITY_NOTIFICATIONS'] = config_obj.get( + 'FEATURE_SECURITY_NOTIFICATIONS', True) + + config_obj['FEATURE_SECURITY_SCANNER'] = config_obj.get( + 'FEATURE_SECURITY_SCANNER', False) + + config_obj['SECURITY_SCANNER_ISSUER_NAME'] = config_obj.get( + 'SECURITY_SCANNER_ISSUER_NAME', 'security_scanner') + # Default mail setings. - config_obj['MAIL_USE_TLS'] = True - config_obj['MAIL_PORT'] = 587 - config_obj['MAIL_DEFAULT_SENDER'] = 'support@quay.io' + config_obj['MAIL_USE_TLS'] = config_obj.get('MAIL_USE_TLS', True) + config_obj['MAIL_PORT'] = config_obj.get('MAIL_PORT', 587) + config_obj['MAIL_DEFAULT_SENDER'] = config_obj.get('MAIL_DEFAULT_SENDER', 'support@quay.io') # Default auth type. if not 'AUTHENTICATION_TYPE' in config_obj: @@ -60,5 +70,5 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname): # Misc configuration. config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http') - config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get('ENTERPRISE_LOGO_URL', - '/static/img/quay-logo.png') + config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get( + 'ENTERPRISE_LOGO_URL', '/static/img/QuayEnterprise_horizontal_color.svg') diff --git a/util/config/validator.py b/util/config/validator.py index 4f8853cb5..a22e91dfe 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -1,10 +1,9 @@ import redis -import os -import json import ldap import peewee import OpenSSL import logging +import time from StringIO import StringIO from fnmatch import fnmatch @@ -14,12 +13,14 @@ from data.users.externalldap import LDAPConnection, LDAPUsers from flask import Flask from flask.ext.mail import Mail, Message -from data.database import validate_database_url, User +from data.database import validate_database_url from storage import get_storage_driver from auth.auth_context import get_authenticated_user from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig from bitbucket import BitBucket from util.security.signing import SIGNING_ENGINES +from util.secscan.api import SecurityScannerAPI +from boot import setup_jwt_proxy from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY @@ -424,6 +425,23 @@ def _validate_signer(config, _): engine.detached_sign(StringIO('test string')) +def _validate_security_scanner(config, _): + """ Validates the configuration for talking to a Quay Security Scanner. """ + # Generate a temporary Quay key to use for signing the outgoing requests. + setup_jwt_proxy() + + # Wait a few seconds for the JWT proxy to startup. + time.sleep(2) + + # Make a ping request to the security service. + client = app.config['HTTPCLIENT'] + api = SecurityScannerAPI(config, None, client=client, skip_validation=True) + response = api.ping() + if response.status_code != 200: + message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text) + raise Exception('Could not ping security scanner: %s' % message) + + _VALIDATORS = { 'database': _validate_database, 'redis': _validate_redis, @@ -439,4 +457,5 @@ _VALIDATORS = { 'jwt': _validate_jwt, 'keystone': _validate_keystone, 'signer': _validate_signer, + 'security-scanner': _validate_security_scanner, } diff --git a/util/secscan/analyzer.py b/util/secscan/analyzer.py index 0558ba808..d178bbff9 100644 --- a/util/secscan/analyzer.py +++ b/util/secscan/analyzer.py @@ -17,10 +17,8 @@ logger = logging.getLogger(__name__) class LayerAnalyzer(object): """ Helper class to perform analysis of a layer via the security scanner. """ def __init__(self, config, api): - secscan_config = config.get('SECURITY_SCANNER') - self._api = api - self._target_version = secscan_config['ENGINE_VERSION_TARGET'] + self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2) def analyze_recursively(self, layer): @@ -62,7 +60,6 @@ class LayerAnalyzer(object): - The second one is set to False when another worker pre-empted the candidate's analysis for us. """ - # If the parent couldn't be analyzed with the target version or higher, we can't analyze # this image. Mark it as failed with the current target version. if (layer.parent_id and not layer.parent.security_indexed and diff --git a/util/secscan/api.py b/util/secscan/api.py index 344865e9c..8c55cb0c4 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -21,26 +21,23 @@ _API_METHOD_INSERT = 'layers' _API_METHOD_GET_LAYER = 'layers/%s' _API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s' _API_METHOD_GET_NOTIFICATION = 'notifications/%s' +_API_METHOD_PING = 'metrics' class SecurityScannerAPI(object): """ Helper class for talking to the Security Scan service (Clair). """ - def __init__(self, config, config_provider, storage): - self.config = config - self.config_provider = config_provider + def __init__(self, config, storage, client=None, skip_validation=False): + if not skip_validation: + config_validator = SecurityConfigValidator(config) + if not config_validator.valid(): + logger.warning('Invalid config provided to SecurityScannerAPI') + return + self._config = config + self._client = client or config['HTTPCLIENT'] self._storage = storage - self._security_config = None - - config_validator = SecurityConfigValidator(config, config_provider) - if not config_validator.valid(): - logger.warning('Invalid config provided to SecurityScannerAPI') - return - self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE'] - - self._security_config = config.get('SECURITY_SCANNER') - self._target_version = self._security_config['ENGINE_VERSION_TARGET'] + self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2) def _get_image_url(self, image): @@ -62,7 +59,7 @@ class SecurityScannerAPI(object): if uri is None: # Handle local storage. local_storage_enabled = False - for storage_type, _ in self.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values(): + for storage_type, _ in self._config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values(): if storage_type == 'LocalStorage': local_storage_enabled = True @@ -99,6 +96,23 @@ class SecurityScannerAPI(object): return request + def ping(self): + """ Calls GET on the metrics endpoint of the security scanner to ensure it is running + and properly configured. Returns the HTTP response. + """ + try: + return self._call('GET', _API_METHOD_PING) + except requests.exceptions.Timeout: + logger.exception('Timeout when trying to connect to security scanner endpoint') + raise Exception('Timeout when trying to connect to security scanner endpoint') + except requests.exceptions.ConnectionError: + logger.exception('Connection error when trying to connect to security scanner endpoint') + raise Exception('Connection error when trying to connect to security scanner endpoint') + except (requests.exceptions.RequestException, ValueError): + logger.exception('Exception when trying to connect to security scanner endpoint') + raise Exception('Exception when trying to connect to security scanner endpoint') + + def analyze_layer(self, layer): """ Posts the given layer to the security scanner for analysis, blocking until complete. Returns a tuple containing the analysis version (on success, None on failure) and @@ -122,6 +136,7 @@ class SecurityScannerAPI(object): logger.exception('Failed to post layer data response for %s', layer.id) return None, False + # Handle any errors from the security scanner. if response.status_code != 201: message = json_response.get('Error').get('Message', '') @@ -235,25 +250,23 @@ class SecurityScannerAPI(object): This function disconnects from the database while awaiting a response from the API server. """ - security_config = self._security_config - if security_config is None: + if self._config is None: raise Exception('Cannot call unconfigured security system') - client = self.config['HTTPCLIENT'] + client = self._client headers = {'Connection': 'close'} - timeout = security_config['API_TIMEOUT_SECONDS'] - endpoint = security_config['ENDPOINT'] + timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 10) + endpoint = self._config['SECURITY_SCANNER_ENDPOINT'] if method != 'GET': - timeout = security_config.get('API_BATCH_TIMEOUT_SECONDS', timeout) - endpoint = security_config.get('ENDPOINT_BATCH', endpoint) + timeout = self._config.get('SECURITY_SCANNER_API_BATCH_TIMEOUT_SECONDS', timeout) + endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT_BATCH') or endpoint - api_url = urljoin(endpoint, '/' + security_config['API_VERSION']) + '/' + api_url = urljoin(endpoint, '/' + self._config.get('SECURITY_SCANNER_API_VERSION', 'v1')) + '/' url = urljoin(api_url, relative_url) - signer_proxy_url = self.config.get('JWTPROXY_SIGNER', 'localhost:8080') + signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080') - - with CloseForLongOperation(self.config): + with CloseForLongOperation(self._config): logger.debug('%sing security URL %s', method.upper(), url) return client.request(method, url, json=body, params=params, timeout=timeout, verify='/conf/mitm.cert', headers=headers, diff --git a/util/secscan/validator.py b/util/secscan/validator.py index bd82cec26..845d8e4b0 100644 --- a/util/secscan/validator.py +++ b/util/secscan/validator.py @@ -6,55 +6,23 @@ logger = logging.getLogger(__name__) class SecurityConfigValidator(object): """ Helper class for validating the security scanner configuration. """ - def __init__(self, config, config_provider): - self._config_provider = config_provider - + def __init__(self, config): if not features.SECURITY_SCANNER: return - self._security_config = config['SECURITY_SCANNER'] - if self._security_config is None: - return - - self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False - self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME') - self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME') - - if self._public_key and self._private_key: - self._keys = (self._public_key, self._private_key) - else: - self._keys = None - - def _get_filepath(self, key): - config = self._security_config - - if key in config: - with self._config_provider.get_volume_file(config[key]) as f: - return f.name - - return None - - def cert(self): - return self._certificate - - def keypair(self): - return self._keys + self._config = config def valid(self): if not features.SECURITY_SCANNER: return False - if not self._security_config: - logger.debug('Missing SECURITY_SCANNER block in configuration') + if self._config.get('SECURITY_SCANNER_ENDPOINT') is None: + logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration') return False - if not 'ENDPOINT' in self._security_config: - logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration') - return False - - endpoint = self._security_config['ENDPOINT'] or '' + endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT') if not endpoint.startswith('http://') and not endpoint.startswith('https://'): - logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https') + logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https') return False return True