Merge pull request #3096 from quay/refactor/secscan-simple

Refactor validators to decouple app
This commit is contained in:
Sam Chow 2018-06-01 15:44:18 -04:00 committed by GitHub
commit 7f21d0da58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
63 changed files with 510 additions and 192 deletions

View file

@ -2,6 +2,8 @@ import os
import re
import subprocess
from util.config.provider import get_config_provider
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
CONF_DIR = os.getenv("QUAYCONF", os.path.join(ROOT_DIR, "conf/"))
@ -10,6 +12,14 @@ STATIC_LDN_DIR = os.path.join(STATIC_DIR, 'ldn/')
STATIC_FONTS_DIR = os.path.join(STATIC_DIR, 'fonts/')
TEMPLATE_DIR = os.path.join(ROOT_DIR, 'templates/')
IS_TESTING = 'TEST' in os.environ
IS_KUBERNETES = 'KUBERNETES_SERVICE_HOST' in os.environ
OVERRIDE_CONFIG_DIRECTORY = os.path.join(CONF_DIR, 'stack/')
config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py',
testing=IS_TESTING, kubernetes=IS_KUBERNETES)
def _get_version_number_changelog():
try:

20
app.py
View file

@ -13,7 +13,9 @@ from flask_principal import Principal
from jwkest.jwk import RSAKey
import features
from _init import CONF_DIR
from _init import config_provider, CONF_DIR, IS_KUBERNETES, IS_TESTING, OVERRIDE_CONFIG_DIRECTORY
from auth.auth_context import get_authenticated_user
from avatars.avatars import Avatar
from buildman.manager.buildcanceller import BuildCanceller
@ -33,15 +35,16 @@ from oauth.services.github import GithubOAuthService
from oauth.services.gitlab import GitLabOAuthService
from oauth.loginmanager import OAuthLoginManager
from storage import Storage
from util.config import URLSchemeAndHostname
from util.log import filter_logs
from util import get_app_url
from util.secscan.secscan_util import get_blob_download_uri_getter
from util.ipresolver import IPResolver
from util.saas.analytics import Analytics
from util.saas.useranalytics import UserAnalytics
from util.saas.exceptionlog import Sentry
from util.names import urn_generator
from util.config.configutil import generate_secret_key
from util.config.provider import get_config_provider
from util.config.superusermanager import SuperUserManager
from util.label_validator import LabelValidator
from util.metrics.metricqueue import MetricQueue
@ -53,7 +56,6 @@ from util.security.instancekeys import InstanceKeys
from util.security.signing import Signer
OVERRIDE_CONFIG_DIRECTORY = os.path.join(CONF_DIR, 'stack/')
OVERRIDE_CONFIG_YAML_FILENAME = os.path.join(CONF_DIR, 'stack/config.yaml')
OVERRIDE_CONFIG_PY_FILENAME = os.path.join(CONF_DIR, 'stack/config.py')
@ -65,10 +67,8 @@ app = Flask(__name__)
logger = logging.getLogger(__name__)
# Instantiate the configuration.
is_testing = 'TEST' in os.environ
is_kubernetes = 'KUBERNETES_SERVICE_HOST' in os.environ
config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py',
testing=is_testing, kubernetes=is_kubernetes)
is_testing = IS_TESTING
is_kubernetes = IS_KUBERNETES
if is_testing:
from test.testconfig import TestConfig
@ -230,7 +230,11 @@ namespace_gc_queue = WorkQueue(app.config['NAMESPACE_GC_QUEUE_NAME'], tf, has_na
all_queues = [image_replication_queue, dockerfile_build_queue, notification_queue,
secscan_notification_queue, chunk_cleanup_queue, namespace_gc_queue]
secscan_api = SecurityScannerAPI(app, app.config, storage)
url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
secscan_api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
instance_keys=instance_keys)
tuf_metadata_api = TUFMetadataAPI(app, app.config)
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.

View file

@ -7,7 +7,7 @@ import subprocess
from flask import abort
from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY
from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver
from auth.permissions import SuperUserPermission
from auth.auth_context import get_authenticated_user
from data.database import configure
@ -20,7 +20,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_
from endpoints.common import common_login
from util.config.configutil import add_enterprise_config_defaults
from util.config.database import sync_database_with_config
from util.config.validator import validate_service_for_config, is_valid_config_upload_filename
from util.config.validator import validate_service_for_config, is_valid_config_upload_filename, ValidatorContext
import features
@ -405,6 +405,10 @@ class SuperUserConfigValidate(ApiResource):
# this is also safe since this method does not access any information not given in the request.
if not config_provider.config_exists() or SuperUserPermission().can():
config = request.get_json()['config']
return validate_service_for_config(service, config, request.get_json().get('password', ''))
validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''),
ip_resolver=ip_resolver,
config_provider=config_provider)
return validate_service_for_config(service, validator_context)
abort(403)

View file

@ -12,7 +12,7 @@ from peewee import IntegrityError
import features
from app import (app, billing as stripe, authentication, avatar, user_analytics, all_queues,
oauth_login, namespace_gc_queue, ip_resolver)
oauth_login, namespace_gc_queue, ip_resolver, url_scheme_and_hostname)
from auth import scopes
from auth.auth_context import get_authenticated_user
@ -785,7 +785,7 @@ class ExternalLoginInformation(ApiResource):
try:
login_scopes = login_service.get_login_scopes()
auth_url = login_service.get_auth_url(app.config, redirect_suffix, csrf_token, login_scopes)
auth_url = login_service.get_auth_url(url_scheme_and_hostname, redirect_suffix, csrf_token, login_scopes)
return {'auth_url': auth_url}
except DiscoveryFailureException as dfe:
logger.exception('Could not discovery OAuth endpoint information')

View file

@ -8,7 +8,7 @@ from peewee import IntegrityError
import features
from app import app, analytics, get_app_url, oauth_login, authentication
from app import app, analytics, get_app_url, oauth_login, authentication, url_scheme_and_hostname
from auth.auth_context import get_authenticated_user
from auth.decorators import require_session_login
from data import model
@ -251,7 +251,7 @@ def _register_service(login_service):
# Redirect to the normal OAuth flow again, so that the user can now create an account.
csrf_token = generate_csrf_token(OAUTH_CSRF_TOKEN_NAME)
login_scopes = login_service.get_login_scopes()
auth_url = login_service.get_auth_url(app.config, '', csrf_token, login_scopes)
auth_url = login_service.get_auth_url(url_scheme_and_hostname, '', csrf_token, login_scopes)
return redirect(auth_url)
@require_session_login

View file

@ -6,7 +6,7 @@ import urlparse
from abc import ABCMeta, abstractmethod
from six import add_metaclass
from util import get_app_url
from util.config import URLSchemeAndHostname
logger = logging.getLogger(__name__)
@ -73,7 +73,7 @@ class OAuthService(object):
pass
@abstractmethod
def validate_client_id_and_secret(self, http_client, app_config):
def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname):
""" Performs validation of the client ID and secret, raising an exception on failure. """
pass
@ -98,9 +98,10 @@ class OAuthService(object):
"""
return self.config.get('LOGIN_BINDING_FIELD', None)
def get_auth_url(self, app_config, redirect_suffix, csrf_token, scopes):
def get_auth_url(self, url_scheme_and_hostname, redirect_suffix, csrf_token, scopes):
""" Retrieves the authorization URL for this login service. """
redirect_uri = '%s/oauth2/%s/callback%s' % (get_app_url(app_config), self.service_id(),
redirect_uri = '%s/oauth2/%s/callback%s' % (url_scheme_and_hostname.get_url(),
self.service_id(),
redirect_suffix)
params = {
'client_id': self.client_id(),
@ -111,9 +112,9 @@ class OAuthService(object):
return self.authorize_endpoint().with_params(params).to_url()
def get_redirect_uri(self, app_config, redirect_suffix=''):
return '%s://%s/oauth2/%s/callback%s' % (app_config['PREFERRED_URL_SCHEME'],
app_config['SERVER_HOSTNAME'],
def get_redirect_uri(self, url_scheme_and_hostname, redirect_suffix=''):
return '%s://%s/oauth2/%s/callback%s' % (url_scheme_and_hostname.url_scheme,
url_scheme_and_hostname.hostname,
self.service_id(),
redirect_suffix)
@ -153,10 +154,11 @@ class OAuthService(object):
def exchange_code(self, app_config, http_client, code, form_encode=False, redirect_suffix='',
client_auth=False):
""" Exchanges an OAuth access code for associated OAuth token and other data. """
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app_config)
payload = {
'code': code,
'grant_type': 'authorization_code',
'redirect_uri': self.get_redirect_uri(app_config, redirect_suffix)
'redirect_uri': self.get_redirect_uri(url_scheme_and_hostname, redirect_suffix)
}
headers = {

View file

@ -97,9 +97,9 @@ class OIDCLoginService(OAuthService):
def validate(self):
return bool(self.get_login_scopes())
def validate_client_id_and_secret(self, http_client, app_config):
def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname):
# TODO: find a way to verify client secret too.
check_auth_url = http_client.get(self.get_auth_url(app_config, '', '', []))
check_auth_url = http_client.get(self.get_auth_url(url_scheme_and_hostname, '', '', []))
if check_auth_url.status_code // 100 != 2:
raise Exception('Got non-200 status code for authorization endpoint')

View file

@ -75,7 +75,7 @@ class GithubOAuthService(OAuthLoginService):
def orgs_endpoint(self):
return slash_join(self._api_endpoint(), 'user/orgs')
def validate_client_id_and_secret(self, http_client, app_config):
def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname):
# First: Verify that the github endpoint is actually Github by checking for the
# X-GitHub-Request-Id here.
api_endpoint = self._api_endpoint()

View file

@ -29,13 +29,13 @@ class GitLabOAuthService(OAuthService):
def token_endpoint(self):
return OAuthEndpoint(slash_join(self._endpoint(), '/oauth/token'))
def validate_client_id_and_secret(self, http_client, app_config):
def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname):
# We validate the client ID and secret by hitting the OAuth token exchange endpoint with
# the real client ID and secret, but a fake auth code to exchange. Gitlab's implementation will
# return `invalid_client` as the `error` if the client ID or secret is invalid; otherwise, it
# will return another error.
url = self.token_endpoint().to_url()
redirect_uri = self.get_redirect_uri(app_config, redirect_suffix='trigger')
redirect_uri = self.get_redirect_uri(url_scheme_and_hostname, redirect_suffix='trigger')
data = {
'code': 'fakecode',
'client_id': self.client_id(),

View file

@ -41,7 +41,7 @@ class GoogleOAuthService(OAuthLoginService):
def requires_form_encoding(self):
return True
def validate_client_id_and_secret(self, http_client, app_config):
def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname):
# To verify the Google client ID and secret, we hit the
# https://www.googleapis.com/oauth2/v3/token endpoint with an invalid request. If the client
# ID or secret are invalid, we get returned a 403 Unauthorized. Otherwise, we get returned

View file

@ -13,6 +13,8 @@ from Crypto.PublicKey import RSA
from jwkest.jwk import RSAKey
from oauth.oidc import OIDCLoginService, OAuthLoginException
from util.config import URLSchemeAndHostname
@pytest.fixture(scope='module') # Slow to generate, only do it once.
def signing_key():
@ -277,7 +279,8 @@ def test_auth_url(oidc_service, discovery_handler, http_client, authorize_handle
config = {'PREFERRED_URL_SCHEME': 'https', 'SERVER_HOSTNAME': 'someserver'}
with HTTMock(discovery_handler, authorize_handler):
auth_url = oidc_service.get_auth_url(config, '', 'some csrf token', ['one', 'two'])
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(config)
auth_url = oidc_service.get_auth_url(url_scheme_and_hostname, '', 'some csrf token', ['one', 'two'])
# Hit the URL and ensure it works.
result = http_client.get(auth_url).json()

View file

@ -2,17 +2,19 @@ import json
import time
import unittest
from app import app, storage, notification_queue
from app import app, storage, notification_queue, url_scheme_and_hostname
from data import model
from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
from endpoints.v2 import v2_bp
from initdb import setup_database_for_testing, finished_database_for_testing
from notifications.notificationevent import VulnerabilityFoundEvent
from util.secscan.secscan_util import get_blob_download_uri_getter
from util.morecollections import AttrDict
from util.secscan.api import SecurityScannerAPI, APIRequestFailure
from util.secscan.analyzer import LayerAnalyzer
from util.secscan.fake import fake_security_scanner
from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult
from util.security.instancekeys import InstanceKeys
from workers.security_notification_worker import SecurityNotificationWorker
@ -42,7 +44,12 @@ class TestSecurityScanner(unittest.TestCase):
self.ctx = app.test_request_context()
self.ctx.__enter__()
self.api = SecurityScannerAPI(app, app.config, storage)
instance_keys = InstanceKeys(app)
self.api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'),
url_scheme_and_hostname),
instance_keys=instance_keys)
def tearDown(self):
storage.remove(['local_us'], 'supports_direct_download')

View file

@ -1,8 +1,8 @@
def get_app_url(config):
""" Returns the application's URL, based on the given config. """
return '%s://%s' % (config['PREFERRED_URL_SCHEME'], config['SERVER_HOSTNAME'])
def slash_join(*args):
"""
Joins together strings and guarantees there is only one '/' in between the
@ -16,3 +16,4 @@ def slash_join(*args):
args = [rmslash(path) for path in args]
return '/'.join(args)

View file

@ -0,0 +1,29 @@
class URLSchemeAndHostname:
"""
Immutable configuration for a given preferred url scheme (e.g. http or https), and a hostname (e.g. localhost:5000)
"""
def __init__(self, url_scheme, hostname):
self._url_scheme = url_scheme
self._hostname = hostname
@classmethod
def from_app_config(cls, app_config):
"""
Helper method to instantiate class from app config, a frequent pattern
:param app_config:
:return:
"""
return cls(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME'])
@property
def url_scheme(self):
return self._url_scheme
@property
def hostname(self):
return self._hostname
def get_url(self):
""" Returns the application's URL, based on the given url scheme and hostname. """
return '%s://%s' % (self._url_scheme, self._hostname)

View file

@ -68,3 +68,7 @@ class BaseFileProvider(BaseProvider):
return True
return False
def get_config_root(self):
return self.config_volume

View file

@ -123,3 +123,8 @@ class BaseProvider(object):
def get_volume_path(self, directory, filename):
""" Helper for constructing relative file paths, which may differ between providers.
For example, kubernetes can't have subfolders in configmaps """
@abstractmethod
def get_config_root(self):
""" Returns the config root directory. """

View file

@ -10,6 +10,10 @@ REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', '
class TestConfigProvider(BaseProvider):
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
the real file system. """
def get_config_root(self):
raise Exception('Test Config does not have a config root')
def __init__(self):
self.clear()

View file

@ -2,6 +2,8 @@ import logging
from auth.auth_context import get_authenticated_user
from data.users import LDAP_CERT_FILENAME
from util.secscan.secscan_util import get_blob_download_uri_getter
from util.config import URLSchemeAndHostname
from util.config.validators.validate_database import DatabaseValidator
from util.config.validators.validate_redis import RedisValidator
@ -64,7 +66,7 @@ VALIDATORS = {
AppTokenAuthValidator.name: AppTokenAuthValidator.validate,
}
def validate_service_for_config(service, config, password=None):
def validate_service_for_config(service, validator_context):
""" Attempts to validate the configuration for the given service. """
if not service in VALIDATORS:
return {
@ -72,7 +74,7 @@ def validate_service_for_config(service, config, password=None):
}
try:
VALIDATORS[service](config, get_authenticated_user(), password)
VALIDATORS[service](validator_context)
return {
'status': True
}
@ -92,3 +94,56 @@ def is_valid_config_upload_filename(filename):
return True
return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES])
class ValidatorContext(object):
""" Context to run validators in, with any additional runtime configuration they need
"""
def __init__(self, config, user_password=None, http_client=None, context=None,
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
uri_creator=None, config_provider=None):
self.config = config
self.user = get_authenticated_user()
self.user_password = user_password
self.http_client = http_client
self.context = context
self.url_scheme_and_hostname = url_scheme_and_hostname
self.jwt_auth_max = jwt_auth_max
self.registry_title = registry_title
self.ip_resolver = ip_resolver
self.feature_sec_scanner = feature_sec_scanner
self.is_testing = is_testing
self.uri_creator = uri_creator
self.config_provider = config_provider
@classmethod
def from_app(cls, app, config, user_password, ip_resolver, client=None, config_provider=None):
"""
Creates a ValidatorContext from an app config, with a given config to validate
:param app: the Flask app to pull configuration information from
:param config: the config to validate
:param user_password: request password
:param ip_resolver: an App
:param client:
:param config_provider:
:return:
"""
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app.config)
cls(config,
user_password,
client or app.config['HTTPCLIENT'],
app.app_context,
url_scheme_and_hostname,
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
app.config['REGISTRY_TITLE'],
ip_resolver,
app.config.get('FEATURE_SECURITY_SCANNER', False),
app.config.get('TESTING', False),
get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
config_provider)

View file

@ -15,6 +15,6 @@ class BaseValidator(object):
@classmethod
@abstractmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Raises Exception if failure to validate. """
pass

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_access import AccessSettingsValidator
@ -17,6 +18,6 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, expected_excepti
if expected_exception is not None:
with pytest.raises(expected_exception):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
@ -12,7 +13,7 @@ from test.fixtures import *
])
def test_skip_validate_actionlog(unvalidated_config, app):
validator = ActionLogArchivingValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('config, expected_error', [
@ -33,19 +34,19 @@ def test_invalid_config(config, expected_error, app):
validator = ActionLogArchivingValidator()
with pytest.raises(ConfigValidationException) as ipe:
validator.validate(config, None, None)
validator.validate(ValidatorContext(config))
assert ipe.value.message == expected_error
def test_valid_config(app):
config = {
config = ValidatorContext({
'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'somepath',
'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
'DISTRIBUTED_STORAGE_CONFIG': {
'somelocation': {},
},
}
})
validator = ActionLogArchivingValidator()
validator.validate(config, None, None)
validator.validate(config)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
@ -15,15 +16,15 @@ def test_validate_invalid_auth_config(unvalidated_config, app):
validator = AppTokenAuthValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_auth(app):
config = {
config = ValidatorContext({
'AUTHENTICATION_TYPE': 'AppToken',
'FEATURE_APP_SPECIFIC_TOKENS': True,
'FEATURE_DIRECT_LOGIN': False,
}
})
validator = AppTokenAuthValidator()
validator.validate(config, None, None)
validator.validate(config)

View file

@ -2,22 +2,24 @@ import pytest
from httmock import urlmatch, HTTMock
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'BITBUCKET_TRIGGER_CONFIG': {}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}}),
(ValidatorContext({})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
])
def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
validator = BitbucketTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(unvalidated_config)
def test_validate_bitbucket_trigger(app):
url_hit = [False]
@ -32,11 +34,15 @@ def test_validate_bitbucket_trigger(app):
with HTTMock(handler):
validator = BitbucketTriggerValidator()
validator.validate({
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'BITBUCKET_TRIGGER_CONFIG': {
'CONSUMER_KEY': 'foo',
'CONSUMER_SECRET': 'bar',
},
}, None, None)
}, url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -1,22 +1,23 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_database import DatabaseValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
(None, None, None, TypeError),
({}, None, None, KeyError),
({'DB_URI': 'sqlite:///:memory:'}, None, None, None),
({'DB_URI': 'invalid:///:memory:'}, None, None, KeyError),
({'DB_NOTURI': 'sqlite:///:memory:'}, None, None, KeyError),
(ValidatorContext(None), None, None, TypeError),
(ValidatorContext({}), None, None, KeyError),
(ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
(ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
(ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
])
def test_validate_database(unvalidated_config, user, user_password, expected, app):
validator = DatabaseValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
@ -36,7 +38,7 @@ def test_validate_invalid_github_config(github_config, github_validator, app):
with pytest.raises(ConfigValidationException):
unvalidated_config = {}
unvalidated_config[github_validator.config_key] = github_config
github_validator.validate(unvalidated_config, None, None)
github_validator.validate(ValidatorContext(unvalidated_config))
def test_validate_github(github_validator, app):
url_hit = [False, False]
@ -52,13 +54,16 @@ def test_validate_github(github_validator, app):
return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
with HTTMock(app_handler, handler):
github_validator.validate({
unvalidated_config = ValidatorContext({
github_validator.config_key: {
'GITHUB_ENDPOINT': 'http://somehost',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
github_validator.validate(unvalidated_config)
assert url_hit[0]
assert url_hit[1]

View file

@ -3,6 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
@ -18,7 +21,7 @@ def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
validator = GitLabTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_gitlab_enterprise_trigger(app):
url_hit = [False]
@ -30,12 +33,17 @@ def test_validate_gitlab_enterprise_trigger(app):
with HTTMock(handler):
validator = GitLabTriggerValidator()
validator.validate({
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'GITLAB_TRIGGER_CONFIG': {
'GITLAB_ENDPOINT': 'http://somegitlab',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
}, http_client=build_requests_session(), url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_google_login import GoogleLoginValidator
@ -17,7 +19,7 @@ def test_validate_invalid_google_login_config(unvalidated_config, app):
validator = GoogleLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_google_login(app):
url_hit = [False]
@ -29,11 +31,15 @@ def test_validate_google_login(app):
validator = GoogleLoginValidator()
with HTTMock(handler):
validator.validate({
unvalidated_config = ValidatorContext({
'GOOGLE_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -1,5 +1,7 @@
import pytest
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_jwt import JWTAuthValidator
from util.morecollections import AttrDict
@ -7,6 +9,7 @@ from util.morecollections import AttrDict
from test.test_external_jwt_authn import fake_jwt
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
@ -14,7 +17,9 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
JWTAuthValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
@ -24,9 +29,13 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
# TODO(jschorr): fix these when re-adding jwt auth mechanism to jwt validators
@pytest.mark.skip(reason='No way of currently testing this')
@pytest.mark.parametrize('username, password, expected_exception', [
('invaliduser', 'invalidpass', ConfigValidationException),
('cool.user', 'invalidpass', ConfigValidationException),
@ -42,10 +51,15 @@ def test_validated_jwt(username, password, expected_exception, app):
config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
unvalidated_config.config_provider = config_provider
unvalidated_config.http_client = build_requests_session()
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
else:
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_keystone import KeystoneValidator
from util.morecollections import AttrDict
@ -13,7 +14,7 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'Keystone'}),
@ -25,7 +26,7 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('username, password, expected_exception', [
@ -45,8 +46,12 @@ def test_validated_keystone(username, password, expected_exception, app):
config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
config['KEYSTONE_ADMIN_TENANT'] = 'admintenant'
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)
else:
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ldap import LDAPValidator
from util.morecollections import AttrDict
@ -7,13 +8,15 @@ from util.morecollections import AttrDict
from test.test_ldap import mock_ldap
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'LDAP'}),
@ -21,7 +24,8 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('uri', [
@ -39,7 +43,8 @@ def test_invalid_uri(uri, app):
config['LDAP_URI'] = uri
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(config, None, None)
config = ValidatorContext(config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('username, password, expected_exception', [
@ -56,10 +61,13 @@ def test_validated_ldap(username, password, expected_exception, app):
config['LDAP_ADMIN_PASSWD'] = 'password'
config['LDAP_USER_RDN'] = ['ou=employees']
unvalidated_config = ValidatorContext(config, user_password=password, config_provider=config_provider)
unvalidated_config.user = AttrDict(dict(username=username))
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)
else:
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)

View file

@ -3,7 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from oauth.oidc import OIDC_WELLKNOWN
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_oidc import OIDCLoginValidator
@ -19,7 +21,7 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, app):
validator = OIDCLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_oidc_login(app):
url_hit = [False]
@ -33,13 +35,16 @@ def test_validate_oidc_login(app):
with HTTMock(handler):
validator = OIDCLoginValidator()
validator.validate({
unvalidated_config = ValidatorContext({
'SOMETHING_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
'OIDC_SERVER': 'http://someserver',
'DEBUGGING': True, # Allows for HTTP.
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -5,10 +5,13 @@ from mock import patch
from mockredis import mock_strict_redis_client
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_redis import RedisValidator
from test.fixtures import *
from util.morecollections import AttrDict
@pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
({}, None, None, False, ConfigValidationException),
@ -19,8 +22,13 @@ from test.fixtures import *
def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
validator = RedisValidator()
unvalidated_config = ValidatorContext(unvalidated_config)
unvalidated_config.user = AttrDict(dict(username=user))
unvalidated_config.user_password = user_password
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

View file

@ -1,6 +1,8 @@
import pytest
from util.config.validators import ConfigValidationException
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators.validate_secscan import SecurityScannerValidator
from util.secscan.fake import fake_security_scanner
@ -10,7 +12,12 @@ from test.fixtures import *
({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
])
def test_validate_noop(unvalidated_config, app):
SecurityScannerValidator.validate(unvalidated_config, None, None)
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
SecurityScannerValidator.validate(unvalidated_config)
@pytest.mark.parametrize('unvalidated_config, expected_error', [
@ -29,9 +36,13 @@ def test_validate_noop(unvalidated_config, app):
}, None),
])
def test_validate(unvalidated_config, expected_error, app):
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
with fake_security_scanner(hostname='fakesecurityscanner'):
if expected_error is not None:
with pytest.raises(expected_error):
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)
else:
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_signer import SignerValidator
@ -14,6 +15,6 @@ def test_validate_signer(unvalidated_config, expected, app):
validator = SignerValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

View file

@ -3,11 +3,13 @@ import pytest
from mock import patch
from tempfile import NamedTemporaryFile
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
from test.test_ssl_util import generate_test_cert
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
@ -16,7 +18,7 @@ from test.fixtures import *
])
def test_skip_validate_ssl(unvalidated_config, app):
validator = SSLValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('cert, expected_error, error_message', [
@ -54,11 +56,13 @@ def test_validate_ssl(cert, expected_error, error_message, app):
with patch('app.config_provider.volume_file_exists', return_true):
with patch('app.config_provider.get_volume_file', get_volume_file):
validator = SSLValidator()
config = ValidatorContext(config)
config.config_provider = config_provider
if expected_error is not None:
with pytest.raises(expected_error) as ipe:
validator.validate(config, None, None)
validator.validate(config)
assert ipe.value.message == error_message
else:
validator.validate(config, None, None)
validator.validate(config)

View file

@ -1,6 +1,7 @@
import moto
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_storage import StorageValidator
@ -16,15 +17,15 @@ def test_validate_storage(unvalidated_config, expected, app):
validator = StorageValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_s3_storage(app):
validator = StorageValidator()
with moto.mock_s3():
with pytest.raises(ConfigValidationException) as ipe:
validator.validate({
validator.validate(ValidatorContext({
'DISTRIBUTED_STORAGE_CONFIG': {
'default': ('S3Storage', {
's3_access_key': 'invalid',
@ -33,6 +34,6 @@ def test_validate_s3_storage(app):
'storage_path': ''
}),
}
}, None, None)
}))
assert ipe.value.message == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_timemachine import TimeMachineValidator
@ -7,7 +8,7 @@ from util.config.validators.validate_timemachine import TimeMachineValidator
({}),
])
def test_validate_noop(unvalidated_config):
TimeMachineValidator.validate(unvalidated_config, None, None)
TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
from test.fixtures import *
@ -25,7 +26,7 @@ def test_validate(default_exp, options, expected_exception, app):
if expected_exception is not None:
with pytest.raises(ConfigValidationException) as cve:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))
assert str(cve.value) == str(expected_exception)
else:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))

View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_torrent import BittorrentValidator
@ -23,8 +25,14 @@ def test_validate_torrent(unvalidated_config, expected, app):
validator = BittorrentValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert not announcer_hit[0]
else:
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert announcer_hit[0]

View file

@ -1,4 +1,3 @@
from app import app
from util.config.validators import BaseValidator, ConfigValidationException
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService
@ -7,13 +6,15 @@ class AccessSettingsValidator(BaseValidator):
name = "access"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
if not config.get('FEATURE_DIRECT_LOGIN', True):
# Make sure we have at least one OIDC enabled.
github_login = config.get('FEATURE_GITHUB_LOGIN', False)
google_login = config.get('FEATURE_GOOGLE_LOGIN', False)
client = app.config['HTTPCLIENT']
login_manager = OAuthLoginManager(config, client=client)
custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)]

View file

@ -4,7 +4,9 @@ class ActionLogArchivingValidator(BaseValidator):
name = "actionlogarchiving"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
""" Validates the action log archiving configuration. """
if not config.get('FEATURE_ACTION_LOG_ROTATION', False):
return

View file

@ -4,7 +4,9 @@ class AppTokenAuthValidator(BaseValidator):
name = "apptoken-auth"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken':
return

View file

@ -1,14 +1,15 @@
from bitbucket import BitBucket
from app import get_app_url
from util.config.validators import BaseValidator, ConfigValidationException
class BitbucketTriggerValidator(BaseValidator):
name = "bitbucket-trigger"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the config for BitBucket. """
config = validator_context.config
trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
if not trigger_config:
raise ConfigValidationException('Missing client ID and client secret')
@ -21,7 +22,7 @@ class BitbucketTriggerValidator(BaseValidator):
key = trigger_config['CONSUMER_KEY']
secret = trigger_config['CONSUMER_SECRET']
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url())
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (validator_context.url_scheme_and_hostname.get_url())
bitbucket_client = BitBucket(key, secret, callback_url)
(result, _, _) = bitbucket_client.get_authorization_url()

View file

@ -7,8 +7,10 @@ class DatabaseValidator(BaseValidator):
name = "database"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates connecting to the database. """
config = validator_context.config
try:
validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
except OperationalError as ex:

View file

@ -1,16 +1,20 @@
from flask import Flask
from flask_mail import Mail, Message
from app import app
from util.config.validators import BaseValidator
class EmailValidator(BaseValidator):
name = "mail"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates sending email. """
with app.app_context():
config = validator_context.config
user = validator_context.user
app_context = validator_context.context
registry_title = validator_context.registry_title
with app_context():
test_app = Flask("mail-test-app")
test_app.config.update(config)
test_app.config.update({
@ -19,7 +23,7 @@ class EmailValidator(BaseValidator):
})
test_mail = Mail(test_app)
test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'],
test_msg = Message("Test e-mail from %s" % registry_title,
sender=config.get('MAIL_DEFAULT_SENDER'))
test_msg.add_recipient(user.email)
test_mail.send(test_msg)

View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.github import GithubOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -7,8 +6,12 @@ class BaseGitHubValidator(BaseValidator):
config_key = None
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a Github service. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
github_config = config.get(cls.config_key)
if not github_config:
raise ConfigValidationException('Missing GitHub client id and client secret')
@ -30,9 +33,8 @@ class BaseGitHubValidator(BaseValidator):
raise ConfigValidationException('Organization restriction must have at least one allowed ' +
'organization')
client = app.config['HTTPCLIENT']
oauth = GithubOAuthService(config, cls.config_key)
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.gitlab import GitLabOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +5,12 @@ class GitLabTriggerValidator(BaseValidator):
name = "gitlab-trigger"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a GitLab service. """
config = validator_context.config
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
client = validator_context.http_client
github_config = config.get('GITLAB_TRIGGER_CONFIG')
if not github_config:
raise ConfigValidationException('Missing GitLab client id and client secret')
@ -23,8 +26,7 @@ class GitLabTriggerValidator(BaseValidator):
if not github_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
client = app.config['HTTPCLIENT']
oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG')
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.google import GoogleOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +5,12 @@ class GoogleLoginValidator(BaseValidator):
name = "google-login"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the Google Login client ID and secret. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
if not google_login_config:
raise ConfigValidationException('Missing client ID and client secret')
@ -18,8 +21,7 @@ class GoogleLoginValidator(BaseValidator):
if not google_login_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
client = app.config['HTTPCLIENT']
oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -1,4 +1,4 @@
from app import app, OVERRIDE_CONFIG_DIRECTORY
import os
from data.users.externaljwt import ExternalJWTAuthN
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +6,15 @@ class JWTAuthValidator(BaseValidator):
name = "jwt"
@classmethod
def validate(cls, config, user, user_password, public_key_path=None):
def validate(cls, validator_context, public_key_path=None):
""" Validates the JWT authentication system. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
http_client = validator_context.http_client
jwt_auth_max = validator_context.jwt_auth_max
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
return
@ -23,12 +30,18 @@ class JWTAuthValidator(BaseValidator):
if not issuer:
raise ConfigValidationException('Missing JWT Issuer ID')
# TODO(jschorr): fix this
return
override_config_directory = os.path.join(config_provider.get_config_root(), '../stack/')
# Try to instatiate the JWT authentication mechanism. This will raise an exception if
# the key cannot be found.
users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
OVERRIDE_CONFIG_DIRECTORY,
app.config['HTTPCLIENT'],
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
override_config_directory,
http_client,
jwt_auth_max,
public_key_path=public_key_path,
requires_email=config.get('FEATURE_MAILING', True))

View file

@ -5,8 +5,12 @@ class KeystoneValidator(BaseValidator):
name = "keystone"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the Keystone authentication system. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
return

View file

@ -2,24 +2,27 @@ import os
import ldap
import subprocess
from app import app, config_provider
from data.users import LDAP_CERT_FILENAME
from data.users.externalldap import LDAPConnection, LDAPUsers
from util.config.validators import BaseValidator, ConfigValidationException
from _init import CONF_DIR
class LDAPValidator(BaseValidator):
name = "ldap"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the LDAP connection. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
return
# If there is a custom LDAP certificate, then reinstall the certificates for the container.
if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
subprocess.check_call([os.path.join(CONF_DIR, 'init/certs_install.sh')])
subprocess.check_call([os.path.join(config_provider.get_config_root(), '../init/certs_install.sh')])
# Note: raises ldap.INVALID_CREDENTIALS on failure
admin_dn = config.get('LDAP_ADMIN_DN')

View file

@ -1,4 +1,3 @@
from app import app
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService, DiscoveryFailureException
from util.config.validators import BaseValidator, ConfigValidationException
@ -7,8 +6,10 @@ class OIDCLoginValidator(BaseValidator):
name = "oidc-login"
@classmethod
def validate(cls, config, user, user_password):
client = app.config['HTTPCLIENT']
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
login_manager = OAuthLoginManager(config, client=client)
for service in login_manager.services:
if not isinstance(service, OIDCLoginService):

View file

@ -6,8 +6,10 @@ class RedisValidator(BaseValidator):
name = "redis"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates connecting to redis. """
config = validator_context.config
redis_config = config.get('BUILDLOGS_REDIS', {})
if not 'host' in redis_config:
raise ConfigValidationException('Missing redis hostname')

View file

@ -1,6 +1,5 @@
import time
from app import app
from boot import setup_jwt_proxy
from util.secscan.api import SecurityScannerAPI
from util.config.validators import BaseValidator, ConfigValidationException
@ -9,15 +8,22 @@ class SecurityScannerValidator(BaseValidator):
name = "security-scanner"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the configuration for talking to a Quay Security Scanner. """
if not config.get('FEATURE_SECURITY_SCANNER', False):
config = validator_context.config
client = validator_context.http_client
feature_sec_scanner = validator_context.feature_sec_scanner
is_testing = validator_context.is_testing
server_hostname = validator_context.url_scheme_and_hostname.hostname
uri_creator = validator_context.uri_creator
if not feature_sec_scanner:
return
client = app.config['HTTPCLIENT']
api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)
api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
if not config.get('TESTING', False):
if not is_testing:
# Generate a temporary Quay key to use for signing the outgoing requests.
setup_jwt_proxy()

View file

@ -1,6 +1,5 @@
from StringIO import StringIO
from app import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.signing import SIGNING_ENGINES
@ -8,8 +7,11 @@ class SignerValidator(BaseValidator):
name = "signer"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the GPG public+private key pair used for signing converted ACIs. """
config = validator_context.config
config_provider = validator_context.config_provider
if config.get('SIGNING_ENGINE') is None:
return

View file

@ -1,4 +1,3 @@
from app import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
@ -8,8 +7,10 @@ class SSLValidator(BaseValidator):
name = "ssl"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the SSL configuration (if enabled). """
config = validator_context.config
config_provider = validator_context.config_provider
# Skip if non-SSL.
if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':

View file

@ -1,16 +1,21 @@
from app import app, ip_resolver, config_provider
from storage import get_storage_driver
from util.config.validators import BaseValidator, ConfigValidationException
class StorageValidator(BaseValidator):
name = "registry-storage"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates registry storage. """
config = validator_context.config
client = validator_context.http_client
ip_resolver = validator_context.ip_resolver
config_provider = validator_context.config_provider
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
providers = _get_storage_providers(config).items()
providers = _get_storage_providers(config, ip_resolver, config_provider).items()
if not providers:
raise ConfigValidationException('Storage configuration required')
@ -21,7 +26,7 @@ class StorageValidator(BaseValidator):
'with storage replication')
# Run validation on the driver.
driver.validate(app.config['HTTPCLIENT'])
driver.validate(client)
# Run setup on the driver if the read/write succeeded.
driver.setup()
@ -30,7 +35,7 @@ class StorageValidator(BaseValidator):
raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
def _get_storage_providers(config):
def _get_storage_providers(config, ip_resolver, config_provider):
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
drivers = {}

View file

@ -9,7 +9,9 @@ class TimeMachineValidator(BaseValidator):
name = "time-machine"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
if not 'DEFAULT_TAG_EXPIRATION' in config:
# Old style config
return

View file

@ -2,7 +2,6 @@ import logging
from hashlib import sha1
from app import app
from util.config.validators import BaseValidator, ConfigValidationException
from util.registry.torrent import jwt_from_infohash
@ -12,15 +11,16 @@ class BittorrentValidator(BaseValidator):
name = "bittorrent"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the configuration for using BitTorrent for downloads. """
config = validator_context.config
client = validator_context.http_client
announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
if not announce_url:
raise ConfigValidationException('Missing announce URL')
# Ensure that the tracker is reachable and accepts requests signed with a registry key.
client = app.config['HTTPCLIENT']
params = {
'info_hash': sha1('test').digest(),
'peer_id': '-QUAY00-6wfG2wk6wWLc',

View file

@ -7,16 +7,12 @@ from urlparse import urljoin
import requests
from flask import url_for
from data.database import CloseForLongOperation
from data import model
from data.model.storage import get_storage_locations
from util import get_app_url, slash_join
from util.abchelpers import nooper
from util.failover import failover, FailoverException
from util.secscan.validator import SecurityConfigValidator
from util.security.instancekeys import InstanceKeys
from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
from _init import CONF_DIR
@ -70,16 +66,16 @@ def compute_layer_id(layer):
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (usually Clair). """
def __init__(self, app, config, storage, client=None, skip_validation=False):
def __init__(self, config, storage, server_hostname=None, client=None, skip_validation=False, uri_creator=None, instance_keys=None):
feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False)
has_valid_config = skip_validation
if not skip_validation and feature_enabled:
config_validator = SecurityConfigValidator(config)
config_validator = SecurityConfigValidator(feature_enabled, config.get('SECURITY_SCANNER_ENDPOINT'))
has_valid_config = config_validator.valid()
if feature_enabled and has_valid_config:
self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client)
self.state = ImplementedSecurityScannerAPI(config, storage, server_hostname, client=client, uri_creator=uri_creator, instance_keys=instance_keys)
else:
self.state = NoopSecurityScannerAPI()
@ -150,20 +146,25 @@ class NoopSecurityScannerAPI(SecurityScannerAPIInterface):
class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
""" Helper class for talking to the Security Scan service (Clair). """
def __init__(self, app, config, storage, client=None):
self._app = app
# TODO(sam) refactor this to not take an app config, and instead just the things it needs as a config object
def __init__(self, config, storage, server_hostname, client=None, uri_creator=None, instance_keys=None):
self._config = config
self._instance_keys = InstanceKeys(app)
self._client = client or config['HTTPCLIENT']
self._instance_keys = instance_keys
self._client = client
self._storage = storage
self._server_hostname = server_hostname
self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
self._uri_creator = uri_creator
def _get_image_url_and_auth(self, image):
""" Returns a tuple of the url and the auth header value that must be used
to fetch the layer data itself. If the image can't be addressed, we return
None.
"""
if self._instance_keys is None:
raise Exception('No Instance keys provided to Security Scanner API')
path = model.storage.get_layer_path(image.storage)
locations = self._default_storage_locations
@ -183,7 +184,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
repository_and_namespace = '/'.join([namespace_name, repo_name])
# Generate the JWT which will authorize this
audience = self._app.config['SERVER_HOSTNAME']
audience = self._server_hostname
context, subject = build_context_and_subject()
access = [{
'type': 'repository',
@ -195,10 +196,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
auth_header = 'Bearer ' + auth_token
with self._app.test_request_context('/'):
relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
digest=image.storage.content_checksum)
uri = urljoin(get_app_url(self._config), relative_layer_url)
uri = self._uri_creator(repository_and_namespace, image.storage.content_checksum)
return uri, auth_header

View file

@ -0,0 +1,22 @@
from urlparse import urljoin
from flask import url_for
def get_blob_download_uri_getter(context, url_scheme_and_hostname):
"""
Returns a function with context to later generate the uri for a download blob
:param context: Flask RequestContext
:param url_scheme_and_hostname: URLSchemeAndHostname class instance
:return: function (repository_and_namespace, checksum) -> uri
"""
def create_uri(repository_and_namespace, checksum):
"""
Creates a uri for a download blob from a repository, namespace, and checksum from earlier context
"""
with context:
relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
digest=checksum)
return urljoin(url_scheme_and_hostname.get_url(), relative_layer_url)
return create_uri

View file

@ -0,0 +1,19 @@
import pytest
from app import app
from util.config import URLSchemeAndHostname
from util.secscan.secscan_util import get_blob_download_uri_getter
from test.fixtures import *
@pytest.mark.parametrize('url_scheme_and_hostname, repo_namespace, checksum, expected_value,', [
(URLSchemeAndHostname('http', 'localhost:5000'),
'devtable/simple', 'tarsum+sha256:123',
'http://localhost:5000/v2/devtable/simple/blobs/tarsum+sha256:123'),
])
def test_blob_download_uri_getter(app, url_scheme_and_hostname,
repo_namespace, checksum,
expected_value):
blob_uri_getter = get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname)
assert blob_uri_getter(repo_namespace, checksum) == expected_value

View file

@ -1,28 +1,27 @@
import logging
import features
logger = logging.getLogger(__name__)
class SecurityConfigValidator(object):
""" Helper class for validating the security scanner configuration. """
def __init__(self, config):
if not features.SECURITY_SCANNER:
def __init__(self, feature_sec_scan, sec_scan_endpoint):
if not feature_sec_scan:
return
self._config = config
self._feature_sec_scan = feature_sec_scan
self._sec_scan_endpoint = sec_scan_endpoint
def valid(self):
if not features.SECURITY_SCANNER:
if not self._feature_sec_scan:
return False
if self._config.get('SECURITY_SCANNER_ENDPOINT') is None:
if self._sec_scan_endpoint is None:
logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
return False
endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT')
endpoint = self._sec_scan_endpoint
if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
return False

View file

@ -19,7 +19,7 @@ DEFAULT_INDEXING_INTERVAL = 30
class SecurityWorker(Worker):
def __init__(self):
super(SecurityWorker, self).__init__()
validator = SecurityConfigValidator(app.config)
validator = SecurityConfigValidator(app.config.get('FEATURE_SECURITY_SCANNER', False), app.config.get('SECURITY_SCANNER_ENDPOINT'))
if not validator.valid():
logger.warning('Failed to validate security scan configuration')
return