Joseph Schorr
parent
53ce4de6aa
commit
2cbdecb043
23 changed files with 584 additions and 116 deletions
|
|
@ -30,10 +30,20 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
|
|||
'signing-public.gpg')
|
||||
config_obj['SIGNING_ENGINE'] = config_obj.get('SIGNING_ENGINE', 'gpg2')
|
||||
|
||||
# Default security scanner config.
|
||||
config_obj['FEATURE_SECURITY_NOTIFICATIONS'] = config_obj.get(
|
||||
'FEATURE_SECURITY_NOTIFICATIONS', True)
|
||||
|
||||
config_obj['FEATURE_SECURITY_SCANNER'] = config_obj.get(
|
||||
'FEATURE_SECURITY_SCANNER', False)
|
||||
|
||||
config_obj['SECURITY_SCANNER_ISSUER_NAME'] = config_obj.get(
|
||||
'SECURITY_SCANNER_ISSUER_NAME', 'security_scanner')
|
||||
|
||||
# Default mail setings.
|
||||
config_obj['MAIL_USE_TLS'] = True
|
||||
config_obj['MAIL_PORT'] = 587
|
||||
config_obj['MAIL_DEFAULT_SENDER'] = 'support@quay.io'
|
||||
config_obj['MAIL_USE_TLS'] = config_obj.get('MAIL_USE_TLS', True)
|
||||
config_obj['MAIL_PORT'] = config_obj.get('MAIL_PORT', 587)
|
||||
config_obj['MAIL_DEFAULT_SENDER'] = config_obj.get('MAIL_DEFAULT_SENDER', 'support@quay.io')
|
||||
|
||||
# Default auth type.
|
||||
if not 'AUTHENTICATION_TYPE' in config_obj:
|
||||
|
|
@ -60,5 +70,5 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
|
|||
|
||||
# Misc configuration.
|
||||
config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http')
|
||||
config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get('ENTERPRISE_LOGO_URL',
|
||||
'/static/img/quay-logo.png')
|
||||
config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get(
|
||||
'ENTERPRISE_LOGO_URL', '/static/img/QuayEnterprise_horizontal_color.svg')
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
import redis
|
||||
import os
|
||||
import json
|
||||
import ldap
|
||||
import peewee
|
||||
import OpenSSL
|
||||
import logging
|
||||
import time
|
||||
|
||||
from StringIO import StringIO
|
||||
from fnmatch import fnmatch
|
||||
|
|
@ -14,12 +13,14 @@ from data.users.externalldap import LDAPConnection, LDAPUsers
|
|||
|
||||
from flask import Flask
|
||||
from flask.ext.mail import Mail, Message
|
||||
from data.database import validate_database_url, User
|
||||
from data.database import validate_database_url
|
||||
from storage import get_storage_driver
|
||||
from auth.auth_context import get_authenticated_user
|
||||
from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
|
||||
from bitbucket import BitBucket
|
||||
from util.security.signing import SIGNING_ENGINES
|
||||
from util.secscan.api import SecurityScannerAPI
|
||||
from boot import setup_jwt_proxy
|
||||
|
||||
from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
|
||||
|
||||
|
|
@ -424,6 +425,23 @@ def _validate_signer(config, _):
|
|||
engine.detached_sign(StringIO('test string'))
|
||||
|
||||
|
||||
def _validate_security_scanner(config, _):
|
||||
""" Validates the configuration for talking to a Quay Security Scanner. """
|
||||
# Generate a temporary Quay key to use for signing the outgoing requests.
|
||||
setup_jwt_proxy()
|
||||
|
||||
# Wait a few seconds for the JWT proxy to startup.
|
||||
time.sleep(2)
|
||||
|
||||
# Make a ping request to the security service.
|
||||
client = app.config['HTTPCLIENT']
|
||||
api = SecurityScannerAPI(config, None, client=client, skip_validation=True)
|
||||
response = api.ping()
|
||||
if response.status_code != 200:
|
||||
message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
|
||||
raise Exception('Could not ping security scanner: %s' % message)
|
||||
|
||||
|
||||
_VALIDATORS = {
|
||||
'database': _validate_database,
|
||||
'redis': _validate_redis,
|
||||
|
|
@ -439,4 +457,5 @@ _VALIDATORS = {
|
|||
'jwt': _validate_jwt,
|
||||
'keystone': _validate_keystone,
|
||||
'signer': _validate_signer,
|
||||
'security-scanner': _validate_security_scanner,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,10 +17,8 @@ logger = logging.getLogger(__name__)
|
|||
class LayerAnalyzer(object):
|
||||
""" Helper class to perform analysis of a layer via the security scanner. """
|
||||
def __init__(self, config, api):
|
||||
secscan_config = config.get('SECURITY_SCANNER')
|
||||
|
||||
self._api = api
|
||||
self._target_version = secscan_config['ENGINE_VERSION_TARGET']
|
||||
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
|
||||
|
||||
|
||||
def analyze_recursively(self, layer):
|
||||
|
|
@ -62,7 +60,6 @@ class LayerAnalyzer(object):
|
|||
- The second one is set to False when another worker pre-empted the candidate's analysis
|
||||
for us.
|
||||
"""
|
||||
|
||||
# If the parent couldn't be analyzed with the target version or higher, we can't analyze
|
||||
# this image. Mark it as failed with the current target version.
|
||||
if (layer.parent_id and not layer.parent.security_indexed and
|
||||
|
|
|
|||
|
|
@ -21,26 +21,23 @@ _API_METHOD_INSERT = 'layers'
|
|||
_API_METHOD_GET_LAYER = 'layers/%s'
|
||||
_API_METHOD_MARK_NOTIFICATION_READ = 'notifications/%s'
|
||||
_API_METHOD_GET_NOTIFICATION = 'notifications/%s'
|
||||
_API_METHOD_PING = 'metrics'
|
||||
|
||||
|
||||
class SecurityScannerAPI(object):
|
||||
""" Helper class for talking to the Security Scan service (Clair). """
|
||||
def __init__(self, config, config_provider, storage):
|
||||
self.config = config
|
||||
self.config_provider = config_provider
|
||||
def __init__(self, config, storage, client=None, skip_validation=False):
|
||||
if not skip_validation:
|
||||
config_validator = SecurityConfigValidator(config)
|
||||
if not config_validator.valid():
|
||||
logger.warning('Invalid config provided to SecurityScannerAPI')
|
||||
return
|
||||
|
||||
self._config = config
|
||||
self._client = client or config['HTTPCLIENT']
|
||||
self._storage = storage
|
||||
self._security_config = None
|
||||
|
||||
config_validator = SecurityConfigValidator(config, config_provider)
|
||||
if not config_validator.valid():
|
||||
logger.warning('Invalid config provided to SecurityScannerAPI')
|
||||
return
|
||||
|
||||
self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
|
||||
|
||||
self._security_config = config.get('SECURITY_SCANNER')
|
||||
self._target_version = self._security_config['ENGINE_VERSION_TARGET']
|
||||
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
|
||||
|
||||
|
||||
def _get_image_url(self, image):
|
||||
|
|
@ -62,7 +59,7 @@ class SecurityScannerAPI(object):
|
|||
if uri is None:
|
||||
# Handle local storage.
|
||||
local_storage_enabled = False
|
||||
for storage_type, _ in self.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
|
||||
for storage_type, _ in self._config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values():
|
||||
if storage_type == 'LocalStorage':
|
||||
local_storage_enabled = True
|
||||
|
||||
|
|
@ -99,6 +96,23 @@ class SecurityScannerAPI(object):
|
|||
return request
|
||||
|
||||
|
||||
def ping(self):
|
||||
""" Calls GET on the metrics endpoint of the security scanner to ensure it is running
|
||||
and properly configured. Returns the HTTP response.
|
||||
"""
|
||||
try:
|
||||
return self._call('GET', _API_METHOD_PING)
|
||||
except requests.exceptions.Timeout:
|
||||
logger.exception('Timeout when trying to connect to security scanner endpoint')
|
||||
raise Exception('Timeout when trying to connect to security scanner endpoint')
|
||||
except requests.exceptions.ConnectionError:
|
||||
logger.exception('Connection error when trying to connect to security scanner endpoint')
|
||||
raise Exception('Connection error when trying to connect to security scanner endpoint')
|
||||
except (requests.exceptions.RequestException, ValueError):
|
||||
logger.exception('Exception when trying to connect to security scanner endpoint')
|
||||
raise Exception('Exception when trying to connect to security scanner endpoint')
|
||||
|
||||
|
||||
def analyze_layer(self, layer):
|
||||
""" Posts the given layer to the security scanner for analysis, blocking until complete.
|
||||
Returns a tuple containing the analysis version (on success, None on failure) and
|
||||
|
|
@ -122,6 +136,7 @@ class SecurityScannerAPI(object):
|
|||
logger.exception('Failed to post layer data response for %s', layer.id)
|
||||
return None, False
|
||||
|
||||
|
||||
# Handle any errors from the security scanner.
|
||||
if response.status_code != 201:
|
||||
message = json_response.get('Error').get('Message', '')
|
||||
|
|
@ -235,25 +250,23 @@ class SecurityScannerAPI(object):
|
|||
This function disconnects from the database while awaiting a response
|
||||
from the API server.
|
||||
"""
|
||||
security_config = self._security_config
|
||||
if security_config is None:
|
||||
if self._config is None:
|
||||
raise Exception('Cannot call unconfigured security system')
|
||||
|
||||
client = self.config['HTTPCLIENT']
|
||||
client = self._client
|
||||
headers = {'Connection': 'close'}
|
||||
|
||||
timeout = security_config['API_TIMEOUT_SECONDS']
|
||||
endpoint = security_config['ENDPOINT']
|
||||
timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 10)
|
||||
endpoint = self._config['SECURITY_SCANNER_ENDPOINT']
|
||||
if method != 'GET':
|
||||
timeout = security_config.get('API_BATCH_TIMEOUT_SECONDS', timeout)
|
||||
endpoint = security_config.get('ENDPOINT_BATCH', endpoint)
|
||||
timeout = self._config.get('SECURITY_SCANNER_API_BATCH_TIMEOUT_SECONDS', timeout)
|
||||
endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT_BATCH') or endpoint
|
||||
|
||||
api_url = urljoin(endpoint, '/' + security_config['API_VERSION']) + '/'
|
||||
api_url = urljoin(endpoint, '/' + self._config.get('SECURITY_SCANNER_API_VERSION', 'v1')) + '/'
|
||||
url = urljoin(api_url, relative_url)
|
||||
signer_proxy_url = self.config.get('JWTPROXY_SIGNER', 'localhost:8080')
|
||||
signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080')
|
||||
|
||||
|
||||
with CloseForLongOperation(self.config):
|
||||
with CloseForLongOperation(self._config):
|
||||
logger.debug('%sing security URL %s', method.upper(), url)
|
||||
return client.request(method, url, json=body, params=params, timeout=timeout,
|
||||
verify='/conf/mitm.cert', headers=headers,
|
||||
|
|
|
|||
|
|
@ -6,55 +6,23 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class SecurityConfigValidator(object):
|
||||
""" Helper class for validating the security scanner configuration. """
|
||||
def __init__(self, config, config_provider):
|
||||
self._config_provider = config_provider
|
||||
|
||||
def __init__(self, config):
|
||||
if not features.SECURITY_SCANNER:
|
||||
return
|
||||
|
||||
self._security_config = config['SECURITY_SCANNER']
|
||||
if self._security_config is None:
|
||||
return
|
||||
|
||||
self._certificate = self._get_filepath('CA_CERTIFICATE_FILENAME') or False
|
||||
self._public_key = self._get_filepath('PUBLIC_KEY_FILENAME')
|
||||
self._private_key = self._get_filepath('PRIVATE_KEY_FILENAME')
|
||||
|
||||
if self._public_key and self._private_key:
|
||||
self._keys = (self._public_key, self._private_key)
|
||||
else:
|
||||
self._keys = None
|
||||
|
||||
def _get_filepath(self, key):
|
||||
config = self._security_config
|
||||
|
||||
if key in config:
|
||||
with self._config_provider.get_volume_file(config[key]) as f:
|
||||
return f.name
|
||||
|
||||
return None
|
||||
|
||||
def cert(self):
|
||||
return self._certificate
|
||||
|
||||
def keypair(self):
|
||||
return self._keys
|
||||
self._config = config
|
||||
|
||||
def valid(self):
|
||||
if not features.SECURITY_SCANNER:
|
||||
return False
|
||||
|
||||
if not self._security_config:
|
||||
logger.debug('Missing SECURITY_SCANNER block in configuration')
|
||||
if self._config.get('SECURITY_SCANNER_ENDPOINT') is None:
|
||||
logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
|
||||
return False
|
||||
|
||||
if not 'ENDPOINT' in self._security_config:
|
||||
logger.debug('Missing ENDPOINT field in SECURITY_SCANNER configuration')
|
||||
return False
|
||||
|
||||
endpoint = self._security_config['ENDPOINT'] or ''
|
||||
endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT')
|
||||
if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
|
||||
logger.debug('ENDPOINT field in SECURITY_SCANNER configuration must start with http or https')
|
||||
logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
|
||||
return False
|
||||
|
||||
return True
|
||||
|
|
|
|||
Reference in a new issue