- Enterprise Registry Setup
+ Quay Enterprise Setup
diff --git a/test/test_api_usage.py b/test/test_api_usage.py
index 0579cbec0..c6a75491c 100644
--- a/test/test_api_usage.py
+++ b/test/test_api_usage.py
@@ -3546,7 +3546,7 @@ class TestRepositoryImageSecurity(ApiTestCase):
# Mark the layer as indexed.
layer.security_indexed = True
- layer.security_indexed_engine = app.config['SECURITY_SCANNER']['ENGINE_VERSION_TARGET']
+ layer.security_indexed_engine = app.config['SECURITY_SCANNER_ENGINE_VERSION_TARGET']
layer.save()
# Grab the security info again.
diff --git a/test/test_secscan.py b/test/test_secscan.py
index 8311d7792..219e262f7 100644
--- a/test/test_secscan.py
+++ b/test/test_secscan.py
@@ -122,7 +122,7 @@ class TestSecurityScanner(unittest.TestCase):
self.ctx = app.test_request_context()
self.ctx.__enter__()
- self.api = SecurityScannerAPI(app.config, config_provider, storage)
+ self.api = SecurityScannerAPI(app.config, storage)
def tearDown(self):
storage.put_content(['local_us'], 'supports_direct_download', 'false')
diff --git a/test/testconfig.py b/test/testconfig.py
index b98da7016..2470e39b1 100644
--- a/test/testconfig.py
+++ b/test/testconfig.py
@@ -61,9 +61,7 @@ class TestConfig(DefaultConfig):
FEATURE_SECURITY_SCANNER = True
FEATURE_SECURITY_NOTIFICATIONS = True
- SECURITY_SCANNER = {
- 'ENDPOINT': 'http://mockclairservice/',
- 'API_VERSION': 'v1',
- 'ENGINE_VERSION_TARGET': 1,
- 'API_TIMEOUT_SECONDS': 1
- }
+ SECURITY_SCANNER_ENDPOINT = 'http://mockclairservice/'
+ SECURITY_SCANNER_API_VERSION = 'v1'
+ SECURITY_SCANNER_ENGINE_VERSION_TARGET = 1
+ SECURITY_SCANNER_API_TIMEOUT_SECONDS = 1
diff --git a/util/config/configutil.py b/util/config/configutil.py
index 80c55f482..969de2a52 100644
--- a/util/config/configutil.py
+++ b/util/config/configutil.py
@@ -30,10 +30,20 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
'signing-public.gpg')
config_obj['SIGNING_ENGINE'] = config_obj.get('SIGNING_ENGINE', 'gpg2')
+ # Default security scanner config.
+ config_obj['FEATURE_SECURITY_NOTIFICATIONS'] = config_obj.get(
+ 'FEATURE_SECURITY_NOTIFICATIONS', True)
+
+ config_obj['FEATURE_SECURITY_SCANNER'] = config_obj.get(
+ 'FEATURE_SECURITY_SCANNER', False)
+
+ config_obj['SECURITY_SCANNER_ISSUER_NAME'] = config_obj.get(
+ 'SECURITY_SCANNER_ISSUER_NAME', 'security_scanner')
+
# Default mail setings.
- config_obj['MAIL_USE_TLS'] = True
- config_obj['MAIL_PORT'] = 587
- config_obj['MAIL_DEFAULT_SENDER'] = 'support@quay.io'
+ config_obj['MAIL_USE_TLS'] = config_obj.get('MAIL_USE_TLS', True)
+ config_obj['MAIL_PORT'] = config_obj.get('MAIL_PORT', 587)
+ config_obj['MAIL_DEFAULT_SENDER'] = config_obj.get('MAIL_DEFAULT_SENDER', 'support@quay.io')
# Default auth type.
if not 'AUTHENTICATION_TYPE' in config_obj:
@@ -60,5 +70,5 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
# Misc configuration.
config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http')
- config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get('ENTERPRISE_LOGO_URL',
- '/static/img/quay-logo.png')
+ config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get(
+ 'ENTERPRISE_LOGO_URL', '/static/img/QuayEnterprise_horizontal_color.svg')
diff --git a/util/config/validator.py b/util/config/validator.py
index 4f8853cb5..a22e91dfe 100644
--- a/util/config/validator.py
+++ b/util/config/validator.py
@@ -1,10 +1,9 @@
import redis
-import os
-import json
import ldap
import peewee
import OpenSSL
import logging
+import time
from StringIO import StringIO
from fnmatch import fnmatch
@@ -14,12 +13,14 @@ from data.users.externalldap import LDAPConnection, LDAPUsers
from flask import Flask
from flask.ext.mail import Mail, Message
-from data.database import validate_database_url, User
+from data.database import validate_database_url
from storage import get_storage_driver
from auth.auth_context import get_authenticated_user
from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
from bitbucket import BitBucket
from util.security.signing import SIGNING_ENGINES
+from util.secscan.api import SecurityScannerAPI
+from boot import setup_jwt_proxy
from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
@@ -424,6 +425,23 @@ def _validate_signer(config, _):
engine.detached_sign(StringIO('test string'))
+def _validate_security_scanner(config, _):
+ """ Validates the configuration for talking to a Quay Security Scanner. """
+ # Generate a temporary Quay key to use for signing the outgoing requests.
+ setup_jwt_proxy()
+
+ # Wait a few seconds for the JWT proxy to startup.
+ time.sleep(2)
+
+ # Make a ping request to the security service.
+ client = app.config['HTTPCLIENT']
+ api = SecurityScannerAPI(config, None, client=client, skip_validation=True)
+ response = api.ping()
+ if response.status_code != 200:
+ message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
+ raise Exception('Could not ping security scanner: %s' % message)
+
+
_VALIDATORS = {
'database': _validate_database,
'redis': _validate_redis,
@@ -439,4 +457,5 @@ _VALIDATORS = {
'jwt': _validate_jwt,
'keystone': _validate_keystone,
'signer': _validate_signer,
+ 'security-scanner': _validate_security_scanner,
}
diff --git a/util/secscan/analyzer.py b/util/secscan/analyzer.py
index 0558ba808..d178bbff9 100644
--- a/util/secscan/analyzer.py
+++ b/util/secscan/analyzer.py
@@ -17,10 +17,8 @@ logger = logging.getLogger(__name__)
class LayerAnalyzer(object):
""" Helper class to perform analysis of a layer via the security scanner. """
def __init__(self, config, api):
- secscan_config = config.get('SECURITY_SCANNER')
-
self._api = api
- self._target_version = secscan_config['ENGINE_VERSION_TARGET']
+ self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
def analyze_recursively(self, layer):
@@ -62,7 +60,6 @@ class LayerAnalyzer(object):
- The second one is set to False when another worker pre-empted the candidate's analysis
for us.
"""
-
# If the parent couldn't be analyzed with the target version or higher, we can't analyze
# this image. Mark it as failed with the current target version.
if (layer.parent_id and not layer.parent.security_indexed and
diff --git a/util/secscan/api.py b/util/secscan/api.py
index 344865e9c..8c55cb0c4 100644
--- a/util/secscan/api.py
+++ b/util/secscan/api.py
@@ -21,26 +21,23 @@ _API_METHOD_INSERT = 'layers'
_API_METHOD_GET_LAYER = 'layers/%s'
_API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s'
_API_METHOD_GET_NOTIFICATION = 'notifications/%s'
+_API_METHOD_PING = 'metrics'
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (Clair). """
- def __init__(self, config, config_provider, storage):
- self.config = config
- self.config_provider = config_provider
+ def __init__(self, config, storage, client=None, skip_validation=False):
+ if not skip_validation:
+ config_validator = SecurityConfigValidator(config)
+ if not config_validator.valid():
+ logger.warning('Invalid config provided to SecurityScannerAPI')
+ return
+ self._config = config
+ self._client = client or config['HTTPCLIENT']
self._storage = storage
- self._security_config = None
-
- config_validator = SecurityConfigValidator(config, config_provider)
- if not config_validator.valid():
- logger.warning('Invalid config provided to SecurityScannerAPI')
- return
-
self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
-
- self._security_config = config.get('SECURITY_SCANNER')
- self._target_version = self._security_config['ENGINE_VERSION_TARGET']
+ self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
def _get_image_url(self, image):
@@ -62,7 +59,7 @@ class SecurityScannerAPI(object):
if uri is None:
# Handle local storage.
local_storage_enabled = False
- for storage_type, _ in self.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
+ for storage_type, _ in self._config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
if storage_type == 'LocalStorage':
local_storage_enabled = True
@@ -99,6 +96,23 @@ class SecurityScannerAPI(object):
return request
+ def ping(self):
+ """ Calls GET on the metrics endpoint of the security scanner to ensure it is running
+ and properly configured. Returns the HTTP response.
+ """
+ try:
+ return self._call('GET', _API_METHOD_PING)
+ except requests.exceptions.Timeout:
+ logger.exception('Timeout when trying to connect to security scanner endpoint')
+ raise Exception('Timeout when trying to connect to security scanner endpoint')
+ except requests.exceptions.ConnectionError:
+ logger.exception('Connection error when trying to connect to security scanner endpoint')
+ raise Exception('Connection error when trying to connect to security scanner endpoint')
+ except (requests.exceptions.RequestException, ValueError):
+ logger.exception('Exception when trying to connect to security scanner endpoint')
+ raise Exception('Exception when trying to connect to security scanner endpoint')
+
+
def analyze_layer(self, layer):
""" Posts the given layer to the security scanner for analysis, blocking until complete.
Returns a tuple containing the analysis version (on success, None on failure) and
@@ -122,6 +136,7 @@ class SecurityScannerAPI(object):
logger.exception('Failed to post layer data response for %s', layer.id)
return None, False
+
# Handle any errors from the security scanner.
if response.status_code != 201:
message = json_response.get('Error').get('Message', '')
@@ -235,25 +250,23 @@ class SecurityScannerAPI(object):
This function disconnects from the database while awaiting a response
from the API server.
"""
- security_config = self._security_config
- if security_config is None:
+ if self._config is None:
raise Exception('Cannot call unconfigured security system')
- client = self.config['HTTPCLIENT']
+ client = self._client
headers = {'Connection': 'close'}
- timeout = security_config['API_TIMEOUT_SECONDS']
- endpoint = security_config['ENDPOINT']
+ timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 10)
+ endpoint = self._config['SECURITY_SCANNER_ENDPOINT']
if method != 'GET':
- timeout = security_config.get('API_BATCH_TIMEOUT_SECONDS', timeout)
- endpoint = security_config.get('ENDPOINT_BATCH', endpoint)
+ timeout = self._config.get('SECURITY_SCANNER_API_BATCH_TIMEOUT_SECONDS', timeout)
+ endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT_BATCH') or endpoint
- api_url = urljoin(endpoint, '/' + security_config['API_VERSION']) + '/'
+ api_url = urljoin(endpoint, '/' + self._config.get('SECURITY_SCANNER_API_VERSION', 'v1')) + '/'
url = urljoin(api_url, relative_url)
- signer_proxy_url = self.config.get('JWTPROXY_SIGNER', 'localhost:8080')
+ signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080')
-
- with CloseForLongOperation(self.config):
+ with CloseForLongOperation(self._config):
logger.debug('%sing security URL %s', method.upper(), url)
return client.request(method, url, json=body, params=params, timeout=timeout,
verify='/conf/mitm.cert', headers=headers,
diff --git a/util/secscan/validator.py b/util/secscan/validator.py
index bd82cec26..845d8e4b0 100644
--- a/util/secscan/validator.py
+++ b/util/secscan/validator.py
@@ -6,55 +6,23 @@ logger = logging.getLogger(__name__)
class SecurityConfigValidator(object):
""" Helper class for validating the security scanner configuration. """
- def __init__(self, config, config_provider):
- self._config_provider = config_provider
-
+ def __init__(self, config):
if not features.SECURITY_SCANNER:
return
- self._security_config = config['SECURITY_SCANNER']
- if self._security_config is None:
- return
-
- self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False
- self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME')
- self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME')
-
- if self._public_key and self._private_key:
- self._keys = (self._public_key, self._private_key)
- else:
- self._keys = None
-
- def _get_filepath(self, key):
- config = self._security_config
-
- if key in config:
- with self._config_provider.get_volume_file(config[key]) as f:
- return f.name
-
- return None
-
- def cert(self):
- return self._certificate
-
- def keypair(self):
- return self._keys
+ self._config = config
def valid(self):
if not features.SECURITY_SCANNER:
return False
- if not self._security_config:
- logger.debug('Missing SECURITY_SCANNER block in configuration')
+ if self._config.get('SECURITY_SCANNER_ENDPOINT') is None:
+ logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
return False
- if not 'ENDPOINT' in self._security_config:
- logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration')
- return False
-
- endpoint = self._security_config['ENDPOINT'] or ''
+ endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT')
if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
- logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https')
+ logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
return False
return True