Merge pull request #3096 from quay/refactor/secscan-simple

Refactor validators to decouple app
This commit is contained in:
Sam Chow 2018-06-01 15:44:18 -04:00 committed by GitHub
commit 7f21d0da58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
63 changed files with 510 additions and 192 deletions

View file

@ -0,0 +1,29 @@
class URLSchemeAndHostname:
"""
Immutable configuration for a given preferred url scheme (e.g. http or https), and a hostname (e.g. localhost:5000)
"""
def __init__(self, url_scheme, hostname):
self._url_scheme = url_scheme
self._hostname = hostname
@classmethod
def from_app_config(cls, app_config):
"""
Helper method to instantiate class from app config, a frequent pattern
:param app_config:
:return:
"""
return cls(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME'])
@property
def url_scheme(self):
return self._url_scheme
@property
def hostname(self):
return self._hostname
def get_url(self):
""" Returns the application's URL, based on the given url scheme and hostname. """
return '%s://%s' % (self._url_scheme, self._hostname)

View file

@ -68,3 +68,7 @@ class BaseFileProvider(BaseProvider):
return True
return False
def get_config_root(self):
return self.config_volume

View file

@ -123,3 +123,8 @@ class BaseProvider(object):
def get_volume_path(self, directory, filename):
""" Helper for constructing relative file paths, which may differ between providers.
For example, kubernetes can't have subfolders in configmaps """
@abstractmethod
def get_config_root(self):
""" Returns the config root directory. """

View file

@ -10,6 +10,10 @@ REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', '
class TestConfigProvider(BaseProvider):
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
the real file system. """
def get_config_root(self):
raise Exception('Test Config does not have a config root')
def __init__(self):
self.clear()

View file

@ -2,6 +2,8 @@ import logging
from auth.auth_context import get_authenticated_user
from data.users import LDAP_CERT_FILENAME
from util.secscan.secscan_util import get_blob_download_uri_getter
from util.config import URLSchemeAndHostname
from util.config.validators.validate_database import DatabaseValidator
from util.config.validators.validate_redis import RedisValidator
@ -64,7 +66,7 @@ VALIDATORS = {
AppTokenAuthValidator.name: AppTokenAuthValidator.validate,
}
def validate_service_for_config(service, config, password=None):
def validate_service_for_config(service, validator_context):
""" Attempts to validate the configuration for the given service. """
if not service in VALIDATORS:
return {
@ -72,7 +74,7 @@ def validate_service_for_config(service, config, password=None):
}
try:
VALIDATORS[service](config, get_authenticated_user(), password)
VALIDATORS[service](validator_context)
return {
'status': True
}
@ -92,3 +94,56 @@ def is_valid_config_upload_filename(filename):
return True
return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES])
class ValidatorContext(object):
""" Context to run validators in, with any additional runtime configuration they need
"""
def __init__(self, config, user_password=None, http_client=None, context=None,
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
uri_creator=None, config_provider=None):
self.config = config
self.user = get_authenticated_user()
self.user_password = user_password
self.http_client = http_client
self.context = context
self.url_scheme_and_hostname = url_scheme_and_hostname
self.jwt_auth_max = jwt_auth_max
self.registry_title = registry_title
self.ip_resolver = ip_resolver
self.feature_sec_scanner = feature_sec_scanner
self.is_testing = is_testing
self.uri_creator = uri_creator
self.config_provider = config_provider
@classmethod
def from_app(cls, app, config, user_password, ip_resolver, client=None, config_provider=None):
"""
Creates a ValidatorContext from an app config, with a given config to validate
:param app: the Flask app to pull configuration information from
:param config: the config to validate
:param user_password: request password
:param ip_resolver: an App
:param client:
:param config_provider:
:return:
"""
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app.config)
cls(config,
user_password,
client or app.config['HTTPCLIENT'],
app.app_context,
url_scheme_and_hostname,
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
app.config['REGISTRY_TITLE'],
ip_resolver,
app.config.get('FEATURE_SECURITY_SCANNER', False),
app.config.get('TESTING', False),
get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
config_provider)

View file

@ -15,6 +15,6 @@ class BaseValidator(object):
@classmethod
@abstractmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Raises Exception if failure to validate. """
pass

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_access import AccessSettingsValidator
@ -17,6 +18,6 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, expected_excepti
if expected_exception is not None:
with pytest.raises(expected_exception):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
@ -12,7 +13,7 @@ from test.fixtures import *
])
def test_skip_validate_actionlog(unvalidated_config, app):
validator = ActionLogArchivingValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('config, expected_error', [
@ -33,19 +34,19 @@ def test_invalid_config(config, expected_error, app):
validator = ActionLogArchivingValidator()
with pytest.raises(ConfigValidationException) as ipe:
validator.validate(config, None, None)
validator.validate(ValidatorContext(config))
assert ipe.value.message == expected_error
def test_valid_config(app):
config = {
config = ValidatorContext({
'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'somepath',
'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
'DISTRIBUTED_STORAGE_CONFIG': {
'somelocation': {},
},
}
})
validator = ActionLogArchivingValidator()
validator.validate(config, None, None)
validator.validate(config)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
@ -15,15 +16,15 @@ def test_validate_invalid_auth_config(unvalidated_config, app):
validator = AppTokenAuthValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_auth(app):
config = {
config = ValidatorContext({
'AUTHENTICATION_TYPE': 'AppToken',
'FEATURE_APP_SPECIFIC_TOKENS': True,
'FEATURE_DIRECT_LOGIN': False,
}
})
validator = AppTokenAuthValidator()
validator.validate(config, None, None)
validator.validate(config)

View file

@ -2,22 +2,24 @@ import pytest
from httmock import urlmatch, HTTMock
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'BITBUCKET_TRIGGER_CONFIG': {}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}}),
(ValidatorContext({})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
])
def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
validator = BitbucketTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(unvalidated_config)
def test_validate_bitbucket_trigger(app):
url_hit = [False]
@ -32,11 +34,15 @@ def test_validate_bitbucket_trigger(app):
with HTTMock(handler):
validator = BitbucketTriggerValidator()
validator.validate({
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'BITBUCKET_TRIGGER_CONFIG': {
'CONSUMER_KEY': 'foo',
'CONSUMER_SECRET': 'bar',
},
}, None, None)
}, url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -1,22 +1,23 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_database import DatabaseValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
(None, None, None, TypeError),
({}, None, None, KeyError),
({'DB_URI': 'sqlite:///:memory:'}, None, None, None),
({'DB_URI': 'invalid:///:memory:'}, None, None, KeyError),
({'DB_NOTURI': 'sqlite:///:memory:'}, None, None, KeyError),
(ValidatorContext(None), None, None, TypeError),
(ValidatorContext({}), None, None, KeyError),
(ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
(ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
(ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
])
def test_validate_database(unvalidated_config, user, user_password, expected, app):
validator = DatabaseValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
@ -36,7 +38,7 @@ def test_validate_invalid_github_config(github_config, github_validator, app):
with pytest.raises(ConfigValidationException):
unvalidated_config = {}
unvalidated_config[github_validator.config_key] = github_config
github_validator.validate(unvalidated_config, None, None)
github_validator.validate(ValidatorContext(unvalidated_config))
def test_validate_github(github_validator, app):
url_hit = [False, False]
@ -52,13 +54,16 @@ def test_validate_github(github_validator, app):
return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
with HTTMock(app_handler, handler):
github_validator.validate({
unvalidated_config = ValidatorContext({
github_validator.config_key: {
'GITHUB_ENDPOINT': 'http://somehost',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
github_validator.validate(unvalidated_config)
assert url_hit[0]
assert url_hit[1]

View file

@ -3,6 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
@ -18,7 +21,7 @@ def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
validator = GitLabTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_gitlab_enterprise_trigger(app):
url_hit = [False]
@ -30,12 +33,17 @@ def test_validate_gitlab_enterprise_trigger(app):
with HTTMock(handler):
validator = GitLabTriggerValidator()
validator.validate({
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'GITLAB_TRIGGER_CONFIG': {
'GITLAB_ENDPOINT': 'http://somegitlab',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
}, http_client=build_requests_session(), url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_google_login import GoogleLoginValidator
@ -17,7 +19,7 @@ def test_validate_invalid_google_login_config(unvalidated_config, app):
validator = GoogleLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_google_login(app):
url_hit = [False]
@ -29,11 +31,15 @@ def test_validate_google_login(app):
validator = GoogleLoginValidator()
with HTTMock(handler):
validator.validate({
unvalidated_config = ValidatorContext({
'GOOGLE_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -1,5 +1,7 @@
import pytest
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_jwt import JWTAuthValidator
from util.morecollections import AttrDict
@ -7,6 +9,7 @@ from util.morecollections import AttrDict
from test.test_external_jwt_authn import fake_jwt
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
@ -14,7 +17,9 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
JWTAuthValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
@ -24,9 +29,13 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
# TODO(jschorr): fix these when re-adding jwt auth mechanism to jwt validators
@pytest.mark.skip(reason='No way of currently testing this')
@pytest.mark.parametrize('username, password, expected_exception', [
('invaliduser', 'invalidpass', ConfigValidationException),
('cool.user', 'invalidpass', ConfigValidationException),
@ -42,10 +51,15 @@ def test_validated_jwt(username, password, expected_exception, app):
config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
unvalidated_config.config_provider = config_provider
unvalidated_config.http_client = build_requests_session()
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
else:
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_keystone import KeystoneValidator
from util.morecollections import AttrDict
@ -13,7 +14,7 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'Keystone'}),
@ -25,7 +26,7 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('username, password, expected_exception', [
@ -45,8 +46,12 @@ def test_validated_keystone(username, password, expected_exception, app):
config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
config['KEYSTONE_ADMIN_TENANT'] = 'admintenant'
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)
else:
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ldap import LDAPValidator
from util.morecollections import AttrDict
@ -7,13 +8,15 @@ from util.morecollections import AttrDict
from test.test_ldap import mock_ldap
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'LDAP'}),
@ -21,7 +24,8 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('uri', [
@ -39,7 +43,8 @@ def test_invalid_uri(uri, app):
config['LDAP_URI'] = uri
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(config, None, None)
config = ValidatorContext(config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('username, password, expected_exception', [
@ -56,10 +61,13 @@ def test_validated_ldap(username, password, expected_exception, app):
config['LDAP_ADMIN_PASSWD'] = 'password'
config['LDAP_USER_RDN'] = ['ou=employees']
unvalidated_config = ValidatorContext(config, user_password=password, config_provider=config_provider)
unvalidated_config.user = AttrDict(dict(username=username))
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)
else:
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)

View file

@ -3,7 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from oauth.oidc import OIDC_WELLKNOWN
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_oidc import OIDCLoginValidator
@ -19,7 +21,7 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, app):
validator = OIDCLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_oidc_login(app):
url_hit = [False]
@ -33,13 +35,16 @@ def test_validate_oidc_login(app):
with HTTMock(handler):
validator = OIDCLoginValidator()
validator.validate({
unvalidated_config = ValidatorContext({
'SOMETHING_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
'OIDC_SERVER': 'http://someserver',
'DEBUGGING': True, # Allows for HTTP.
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -5,10 +5,13 @@ from mock import patch
from mockredis import mock_strict_redis_client
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_redis import RedisValidator
from test.fixtures import *
from util.morecollections import AttrDict
@pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
({}, None, None, False, ConfigValidationException),
@ -19,8 +22,13 @@ from test.fixtures import *
def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
validator = RedisValidator()
unvalidated_config = ValidatorContext(unvalidated_config)
unvalidated_config.user = AttrDict(dict(username=user))
unvalidated_config.user_password = user_password
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

View file

@ -1,6 +1,8 @@
import pytest
from util.config.validators import ConfigValidationException
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators.validate_secscan import SecurityScannerValidator
from util.secscan.fake import fake_security_scanner
@ -10,7 +12,12 @@ from test.fixtures import *
({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
])
def test_validate_noop(unvalidated_config, app):
SecurityScannerValidator.validate(unvalidated_config, None, None)
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
SecurityScannerValidator.validate(unvalidated_config)
@pytest.mark.parametrize('unvalidated_config, expected_error', [
@ -29,9 +36,13 @@ def test_validate_noop(unvalidated_config, app):
}, None),
])
def test_validate(unvalidated_config, expected_error, app):
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
with fake_security_scanner(hostname='fakesecurityscanner'):
if expected_error is not None:
with pytest.raises(expected_error):
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)
else:
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_signer import SignerValidator
@ -14,6 +15,6 @@ def test_validate_signer(unvalidated_config, expected, app):
validator = SignerValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

View file

@ -3,11 +3,13 @@ import pytest
from mock import patch
from tempfile import NamedTemporaryFile
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
from test.test_ssl_util import generate_test_cert
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
@ -16,7 +18,7 @@ from test.fixtures import *
])
def test_skip_validate_ssl(unvalidated_config, app):
validator = SSLValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('cert, expected_error, error_message', [
@ -54,11 +56,13 @@ def test_validate_ssl(cert, expected_error, error_message, app):
with patch('app.config_provider.volume_file_exists', return_true):
with patch('app.config_provider.get_volume_file', get_volume_file):
validator = SSLValidator()
config = ValidatorContext(config)
config.config_provider = config_provider
if expected_error is not None:
with pytest.raises(expected_error) as ipe:
validator.validate(config, None, None)
validator.validate(config)
assert ipe.value.message == error_message
else:
validator.validate(config, None, None)
validator.validate(config)

View file

@ -1,6 +1,7 @@
import moto
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_storage import StorageValidator
@ -16,15 +17,15 @@ def test_validate_storage(unvalidated_config, expected, app):
validator = StorageValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_s3_storage(app):
validator = StorageValidator()
with moto.mock_s3():
with pytest.raises(ConfigValidationException) as ipe:
validator.validate({
validator.validate(ValidatorContext({
'DISTRIBUTED_STORAGE_CONFIG': {
'default': ('S3Storage', {
's3_access_key': 'invalid',
@ -33,6 +34,6 @@ def test_validate_s3_storage(app):
'storage_path': ''
}),
}
}, None, None)
}))
assert ipe.value.message == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'

View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_timemachine import TimeMachineValidator
@ -7,7 +8,7 @@ from util.config.validators.validate_timemachine import TimeMachineValidator
({}),
])
def test_validate_noop(unvalidated_config):
TimeMachineValidator.validate(unvalidated_config, None, None)
TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
from test.fixtures import *
@ -25,7 +26,7 @@ def test_validate(default_exp, options, expected_exception, app):
if expected_exception is not None:
with pytest.raises(ConfigValidationException) as cve:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))
assert str(cve.value) == str(expected_exception)
else:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))

View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_torrent import BittorrentValidator
@ -23,8 +25,14 @@ def test_validate_torrent(unvalidated_config, expected, app):
validator = BittorrentValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert not announcer_hit[0]
else:
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert announcer_hit[0]

View file

@ -1,4 +1,3 @@
from app import app
from util.config.validators import BaseValidator, ConfigValidationException
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService
@ -7,13 +6,15 @@ class AccessSettingsValidator(BaseValidator):
name = "access"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
if not config.get('FEATURE_DIRECT_LOGIN', True):
# Make sure we have at least one OIDC enabled.
github_login = config.get('FEATURE_GITHUB_LOGIN', False)
google_login = config.get('FEATURE_GOOGLE_LOGIN', False)
client = app.config['HTTPCLIENT']
login_manager = OAuthLoginManager(config, client=client)
custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)]

View file

@ -4,7 +4,9 @@ class ActionLogArchivingValidator(BaseValidator):
name = "actionlogarchiving"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
""" Validates the action log archiving configuration. """
if not config.get('FEATURE_ACTION_LOG_ROTATION', False):
return

View file

@ -4,7 +4,9 @@ class AppTokenAuthValidator(BaseValidator):
name = "apptoken-auth"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken':
return

View file

@ -1,14 +1,15 @@
from bitbucket import BitBucket
from app import get_app_url
from util.config.validators import BaseValidator, ConfigValidationException
class BitbucketTriggerValidator(BaseValidator):
name = "bitbucket-trigger"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the config for BitBucket. """
config = validator_context.config
trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
if not trigger_config:
raise ConfigValidationException('Missing client ID and client secret')
@ -21,7 +22,7 @@ class BitbucketTriggerValidator(BaseValidator):
key = trigger_config['CONSUMER_KEY']
secret = trigger_config['CONSUMER_SECRET']
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url())
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (validator_context.url_scheme_and_hostname.get_url())
bitbucket_client = BitBucket(key, secret, callback_url)
(result, _, _) = bitbucket_client.get_authorization_url()

View file

@ -7,8 +7,10 @@ class DatabaseValidator(BaseValidator):
name = "database"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates connecting to the database. """
config = validator_context.config
try:
validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
except OperationalError as ex:

View file

@ -1,16 +1,20 @@
from flask import Flask
from flask_mail import Mail, Message
from app import app
from util.config.validators import BaseValidator
class EmailValidator(BaseValidator):
name = "mail"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates sending email. """
with app.app_context():
config = validator_context.config
user = validator_context.user
app_context = validator_context.context
registry_title = validator_context.registry_title
with app_context():
test_app = Flask("mail-test-app")
test_app.config.update(config)
test_app.config.update({
@ -19,7 +23,7 @@ class EmailValidator(BaseValidator):
})
test_mail = Mail(test_app)
test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'],
test_msg = Message("Test e-mail from %s" % registry_title,
sender=config.get('MAIL_DEFAULT_SENDER'))
test_msg.add_recipient(user.email)
test_mail.send(test_msg)

View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.github import GithubOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -7,8 +6,12 @@ class BaseGitHubValidator(BaseValidator):
config_key = None
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a Github service. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
github_config = config.get(cls.config_key)
if not github_config:
raise ConfigValidationException('Missing GitHub client id and client secret')
@ -30,9 +33,8 @@ class BaseGitHubValidator(BaseValidator):
raise ConfigValidationException('Organization restriction must have at least one allowed ' +
'organization')
client = app.config['HTTPCLIENT']
oauth = GithubOAuthService(config, cls.config_key)
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.gitlab import GitLabOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +5,12 @@ class GitLabTriggerValidator(BaseValidator):
name = "gitlab-trigger"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a GitLab service. """
config = validator_context.config
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
client = validator_context.http_client
github_config = config.get('GITLAB_TRIGGER_CONFIG')
if not github_config:
raise ConfigValidationException('Missing GitLab client id and client secret')
@ -23,8 +26,7 @@ class GitLabTriggerValidator(BaseValidator):
if not github_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
client = app.config['HTTPCLIENT']
oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG')
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.google import GoogleOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +5,12 @@ class GoogleLoginValidator(BaseValidator):
name = "google-login"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the Google Login client ID and secret. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
if not google_login_config:
raise ConfigValidationException('Missing client ID and client secret')
@ -18,8 +21,7 @@ class GoogleLoginValidator(BaseValidator):
if not google_login_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
client = app.config['HTTPCLIENT']
oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -1,4 +1,4 @@
from app import app, OVERRIDE_CONFIG_DIRECTORY
import os
from data.users.externaljwt import ExternalJWTAuthN
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +6,15 @@ class JWTAuthValidator(BaseValidator):
name = "jwt"
@classmethod
def validate(cls, config, user, user_password, public_key_path=None):
def validate(cls, validator_context, public_key_path=None):
""" Validates the JWT authentication system. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
http_client = validator_context.http_client
jwt_auth_max = validator_context.jwt_auth_max
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
return
@ -23,12 +30,18 @@ class JWTAuthValidator(BaseValidator):
if not issuer:
raise ConfigValidationException('Missing JWT Issuer ID')
# TODO(jschorr): fix this
return
override_config_directory = os.path.join(config_provider.get_config_root(), '../stack/')
# Try to instatiate the JWT authentication mechanism. This will raise an exception if
# the key cannot be found.
users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
OVERRIDE_CONFIG_DIRECTORY,
app.config['HTTPCLIENT'],
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
override_config_directory,
http_client,
jwt_auth_max,
public_key_path=public_key_path,
requires_email=config.get('FEATURE_MAILING', True))

View file

@ -5,8 +5,12 @@ class KeystoneValidator(BaseValidator):
name = "keystone"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the Keystone authentication system. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
return

View file

@ -2,24 +2,27 @@ import os
import ldap
import subprocess
from app import app, config_provider
from data.users import LDAP_CERT_FILENAME
from data.users.externalldap import LDAPConnection, LDAPUsers
from util.config.validators import BaseValidator, ConfigValidationException
from _init import CONF_DIR
class LDAPValidator(BaseValidator):
name = "ldap"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the LDAP connection. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
return
# If there is a custom LDAP certificate, then reinstall the certificates for the container.
if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
subprocess.check_call([os.path.join(CONF_DIR, 'init/certs_install.sh')])
subprocess.check_call([os.path.join(config_provider.get_config_root(), '../init/certs_install.sh')])
# Note: raises ldap.INVALID_CREDENTIALS on failure
admin_dn = config.get('LDAP_ADMIN_DN')

View file

@ -1,4 +1,3 @@
from app import app
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService, DiscoveryFailureException
from util.config.validators import BaseValidator, ConfigValidationException
@ -7,8 +6,10 @@ class OIDCLoginValidator(BaseValidator):
name = "oidc-login"
@classmethod
def validate(cls, config, user, user_password):
client = app.config['HTTPCLIENT']
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
login_manager = OAuthLoginManager(config, client=client)
for service in login_manager.services:
if not isinstance(service, OIDCLoginService):

View file

@ -6,8 +6,10 @@ class RedisValidator(BaseValidator):
name = "redis"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates connecting to redis. """
config = validator_context.config
redis_config = config.get('BUILDLOGS_REDIS', {})
if not 'host' in redis_config:
raise ConfigValidationException('Missing redis hostname')

View file

@ -1,6 +1,5 @@
import time
from app import app
from boot import setup_jwt_proxy
from util.secscan.api import SecurityScannerAPI
from util.config.validators import BaseValidator, ConfigValidationException
@ -9,15 +8,22 @@ class SecurityScannerValidator(BaseValidator):
name = "security-scanner"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the configuration for talking to a Quay Security Scanner. """
if not config.get('FEATURE_SECURITY_SCANNER', False):
config = validator_context.config
client = validator_context.http_client
feature_sec_scanner = validator_context.feature_sec_scanner
is_testing = validator_context.is_testing
server_hostname = validator_context.url_scheme_and_hostname.hostname
uri_creator = validator_context.uri_creator
if not feature_sec_scanner:
return
client = app.config['HTTPCLIENT']
api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)
api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
if not config.get('TESTING', False):
if not is_testing:
# Generate a temporary Quay key to use for signing the outgoing requests.
setup_jwt_proxy()

View file

@ -1,6 +1,5 @@
from StringIO import StringIO
from app import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.signing import SIGNING_ENGINES
@ -8,8 +7,11 @@ class SignerValidator(BaseValidator):
name = "signer"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the GPG public+private key pair used for signing converted ACIs. """
config = validator_context.config
config_provider = validator_context.config_provider
if config.get('SIGNING_ENGINE') is None:
return

View file

@ -1,4 +1,3 @@
from app import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
@ -8,8 +7,10 @@ class SSLValidator(BaseValidator):
name = "ssl"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the SSL configuration (if enabled). """
config = validator_context.config
config_provider = validator_context.config_provider
# Skip if non-SSL.
if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':

View file

@ -1,16 +1,21 @@
from app import app, ip_resolver, config_provider
from storage import get_storage_driver
from util.config.validators import BaseValidator, ConfigValidationException
class StorageValidator(BaseValidator):
name = "registry-storage"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates registry storage. """
config = validator_context.config
client = validator_context.http_client
ip_resolver = validator_context.ip_resolver
config_provider = validator_context.config_provider
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
providers = _get_storage_providers(config).items()
providers = _get_storage_providers(config, ip_resolver, config_provider).items()
if not providers:
raise ConfigValidationException('Storage configuration required')
@ -21,7 +26,7 @@ class StorageValidator(BaseValidator):
'with storage replication')
# Run validation on the driver.
driver.validate(app.config['HTTPCLIENT'])
driver.validate(client)
# Run setup on the driver if the read/write succeeded.
driver.setup()
@ -30,7 +35,7 @@ class StorageValidator(BaseValidator):
raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
def _get_storage_providers(config):
def _get_storage_providers(config, ip_resolver, config_provider):
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
drivers = {}

View file

@ -9,7 +9,9 @@ class TimeMachineValidator(BaseValidator):
name = "time-machine"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
if not 'DEFAULT_TAG_EXPIRATION' in config:
# Old style config
return

View file

@ -2,7 +2,6 @@ import logging
from hashlib import sha1
from app import app
from util.config.validators import BaseValidator, ConfigValidationException
from util.registry.torrent import jwt_from_infohash
@ -12,15 +11,16 @@ class BittorrentValidator(BaseValidator):
name = "bittorrent"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the configuration for using BitTorrent for downloads. """
config = validator_context.config
client = validator_context.http_client
announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
if not announce_url:
raise ConfigValidationException('Missing announce URL')
# Ensure that the tracker is reachable and accepts requests signed with a registry key.
client = app.config['HTTPCLIENT']
params = {
'info_hash': sha1('test').digest(),
'peer_id': '-QUAY00-6wfG2wk6wWLc',