vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3096 from quay/refactor/secscan-simple

Browse source 
Refactor validators to decouple app
This commit is contained in:
Sam Chow 2018-06-01 15:44:18 -04:00 • committed by GitHub
commit 7f21d0da58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
63 changed files with 510 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

3
util/__init__.py
View file

@ -1,8 +1,8 @@
def get_app_url(config):
""" Returns the application's URL, based on the given config. """
return '%s://%s' % (config['PREFERRED_URL_SCHEME'], config['SERVER_HOSTNAME'])
def slash_join(*args):
"""
Joins together strings and guarantees there is only one '/' in between the
@ -16,3 +16,4 @@ def slash_join(*args):
args = [rmslash(path) for path in args]
return '/'.join(args)

29
util/config/__init__.py
View file

@ -0,0 +1,29 @@
class URLSchemeAndHostname:
"""
Immutable configuration for a given preferred url scheme (e.g. http or https), and a hostname (e.g. localhost:5000)
"""
def __init__(self, url_scheme, hostname):
self._url_scheme = url_scheme
self._hostname = hostname
@classmethod
def from_app_config(cls, app_config):
"""
Helper method to instantiate class from app config, a frequent pattern
:param app_config:
:return:
"""
return cls(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME'])
@property
def url_scheme(self):
return self._url_scheme
@property
def hostname(self):
return self._hostname
def get_url(self):
""" Returns the application's URL, based on the given url scheme and hostname. """
return '%s://%s' % (self._url_scheme, self._hostname)

4
util/config/provider/basefileprovider.py
View file

@ -68,3 +68,7 @@ class BaseFileProvider(BaseProvider):
return True
return False
def get_config_root(self):
return self.config_volume

5
util/config/provider/baseprovider.py
View file

@ -123,3 +123,8 @@ class BaseProvider(object):
def get_volume_path(self, directory, filename):
""" Helper for constructing relative file paths, which may differ between providers.
For example, kubernetes can't have subfolders in configmaps """
@abstractmethod
def get_config_root(self):
""" Returns the config root directory. """

4
util/config/provider/testprovider.py
View file

@ -10,6 +10,10 @@ REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', '
class TestConfigProvider(BaseProvider):
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
the real file system. """
def get_config_root(self):
raise Exception('Test Config does not have a config root')
def __init__(self):
self.clear()

59
util/config/validator.py
View file

@ -2,6 +2,8 @@ import logging
from auth.auth_context import get_authenticated_user
from data.users import LDAP_CERT_FILENAME
from util.secscan.secscan_util import get_blob_download_uri_getter
from util.config import URLSchemeAndHostname
from util.config.validators.validate_database import DatabaseValidator
from util.config.validators.validate_redis import RedisValidator
@ -64,7 +66,7 @@ VALIDATORS = {
AppTokenAuthValidator.name: AppTokenAuthValidator.validate,
}
def validate_service_for_config(service, config, password=None):
def validate_service_for_config(service, validator_context):
""" Attempts to validate the configuration for the given service. """
if not service in VALIDATORS:
return {
@ -72,7 +74,7 @@ def validate_service_for_config(service, config, password=None):
}
try:
VALIDATORS[service](config, get_authenticated_user(), password)
VALIDATORS[service](validator_context)
return {
'status': True
}
@ -92,3 +94,56 @@ def is_valid_config_upload_filename(filename):
return True
return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES])
class ValidatorContext(object):
""" Context to run validators in, with any additional runtime configuration they need
"""
def __init__(self, config, user_password=None, http_client=None, context=None,
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
uri_creator=None, config_provider=None):
self.config = config
self.user = get_authenticated_user()
self.user_password = user_password
self.http_client = http_client
self.context = context
self.url_scheme_and_hostname = url_scheme_and_hostname
self.jwt_auth_max = jwt_auth_max
self.registry_title = registry_title
self.ip_resolver = ip_resolver
self.feature_sec_scanner = feature_sec_scanner
self.is_testing = is_testing
self.uri_creator = uri_creator
self.config_provider = config_provider
@classmethod
def from_app(cls, app, config, user_password, ip_resolver, client=None, config_provider=None):
"""
Creates a ValidatorContext from an app config, with a given config to validate
:param app: the Flask app to pull configuration information from
:param config: the config to validate
:param user_password: request password
:param ip_resolver: an App
:param client:
:param config_provider:
:return:
"""
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app.config)
cls(config,
user_password,
client or app.config['HTTPCLIENT'],
app.app_context,
url_scheme_and_hostname,
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
app.config['REGISTRY_TITLE'],
ip_resolver,
app.config.get('FEATURE_SECURITY_SCANNER', False),
app.config.get('TESTING', False),
get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
config_provider)

2
util/config/validators/__init__.py
View file

@ -15,6 +15,6 @@ class BaseValidator(object):
@classmethod
@abstractmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Raises Exception if failure to validate. """
pass

5
util/config/validators/test/test_validate_access.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_access import AccessSettingsValidator
@ -17,6 +18,6 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, expected_excepti
if expected_exception is not None:
with pytest.raises(expected_exception):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

11
util/config/validators/test/test_validate_actionlog_archiving.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
@ -12,7 +13,7 @@ from test.fixtures import *
])
def test_skip_validate_actionlog(unvalidated_config, app):
validator = ActionLogArchivingValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('config, expected_error', [
@ -33,19 +34,19 @@ def test_invalid_config(config, expected_error, app):
validator = ActionLogArchivingValidator()
with pytest.raises(ConfigValidationException) as ipe:
validator.validate(config, None, None)
validator.validate(ValidatorContext(config))
assert ipe.value.message == expected_error
def test_valid_config(app):
config = {
config = ValidatorContext({
'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'somepath',
'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
'DISTRIBUTED_STORAGE_CONFIG': {
'somelocation': {},
},
}
})
validator = ActionLogArchivingValidator()
validator.validate(config, None, None)
validator.validate(config)

9
util/config/validators/test/test_validate_apptokenauth.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
@ -15,15 +16,15 @@ def test_validate_invalid_auth_config(unvalidated_config, app):
validator = AppTokenAuthValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_auth(app):
config = {
config = ValidatorContext({
'AUTHENTICATION_TYPE': 'AppToken',
'FEATURE_APP_SPECIFIC_TOKENS': True,
'FEATURE_DIRECT_LOGIN': False,
}
})
validator = AppTokenAuthValidator()
validator.validate(config, None, None)
validator.validate(config)

20
util/config/validators/test/test_validate_bitbucket_trigger.py
View file

@ -2,22 +2,24 @@ import pytest
from httmock import urlmatch, HTTMock
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'BITBUCKET_TRIGGER_CONFIG': {}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}}),
(ValidatorContext({})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
])
def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
validator = BitbucketTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(unvalidated_config)
def test_validate_bitbucket_trigger(app):
url_hit = [False]
@ -32,11 +34,15 @@ def test_validate_bitbucket_trigger(app):
with HTTMock(handler):
validator = BitbucketTriggerValidator()
validator.validate({
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'BITBUCKET_TRIGGER_CONFIG': {
'CONSUMER_KEY': 'foo',
'CONSUMER_SECRET': 'bar',
},
}, None, None)
}, url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

15
util/config/validators/test/test_validate_database.py
View file

@ -1,22 +1,23 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_database import DatabaseValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
(None, None, None, TypeError),
({}, None, None, KeyError),
({'DB_URI': 'sqlite:///:memory:'}, None, None, None),
({'DB_URI': 'invalid:///:memory:'}, None, None, KeyError),
({'DB_NOTURI': 'sqlite:///:memory:'}, None, None, KeyError),
(ValidatorContext(None), None, None, TypeError),
(ValidatorContext({}), None, None, KeyError),
(ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
(ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
(ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
])
def test_validate_database(unvalidated_config, user, user_password, expected, app):
validator = DatabaseValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

11
util/config/validators/test/test_validate_github.py
View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
@ -36,7 +38,7 @@ def test_validate_invalid_github_config(github_config, github_validator, app):
with pytest.raises(ConfigValidationException):
unvalidated_config = {}
unvalidated_config[github_validator.config_key] = github_config
github_validator.validate(unvalidated_config, None, None)
github_validator.validate(ValidatorContext(unvalidated_config))
def test_validate_github(github_validator, app):
url_hit = [False, False]
@ -52,13 +54,16 @@ def test_validate_github(github_validator, app):
return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
with HTTMock(app_handler, handler):
github_validator.validate({
unvalidated_config = ValidatorContext({
github_validator.config_key: {
'GITHUB_ENDPOINT': 'http://somehost',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
github_validator.validate(unvalidated_config)
assert url_hit[0]
assert url_hit[1]

14
util/config/validators/test/test_validate_gitlab_trigger.py
View file

@ -3,6 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
@ -18,7 +21,7 @@ def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
validator = GitLabTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_gitlab_enterprise_trigger(app):
url_hit = [False]
@ -30,12 +33,17 @@ def test_validate_gitlab_enterprise_trigger(app):
with HTTMock(handler):
validator = GitLabTriggerValidator()
validator.validate({
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'GITLAB_TRIGGER_CONFIG': {
'GITLAB_ENDPOINT': 'http://somegitlab',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
}, http_client=build_requests_session(), url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

12
util/config/validators/test/test_validate_google_login.py
View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_google_login import GoogleLoginValidator
@ -17,7 +19,7 @@ def test_validate_invalid_google_login_config(unvalidated_config, app):
validator = GoogleLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_google_login(app):
url_hit = [False]
@ -29,11 +31,15 @@ def test_validate_google_login(app):
validator = GoogleLoginValidator()
with HTTMock(handler):
validator.validate({
unvalidated_config = ValidatorContext({
'GOOGLE_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

26
util/config/validators/test/test_validate_jwt.py
View file

@ -1,5 +1,7 @@
import pytest
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_jwt import JWTAuthValidator
from util.morecollections import AttrDict
@ -7,6 +9,7 @@ from util.morecollections import AttrDict
from test.test_external_jwt_authn import fake_jwt
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
@ -14,7 +17,9 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
JWTAuthValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
@ -24,9 +29,13 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
# TODO(jschorr): fix these when re-adding jwt auth mechanism to jwt validators
@pytest.mark.skip(reason='No way of currently testing this')
@pytest.mark.parametrize('username, password, expected_exception', [
('invaliduser', 'invalidpass', ConfigValidationException),
('cool.user', 'invalidpass', ConfigValidationException),
@ -42,10 +51,15 @@ def test_validated_jwt(username, password, expected_exception, app):
config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
unvalidated_config.config_provider = config_provider
unvalidated_config.http_client = build_requests_session()
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
else:
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)

13
util/config/validators/test/test_validate_keystone.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_keystone import KeystoneValidator
from util.morecollections import AttrDict
@ -13,7 +14,7 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'Keystone'}),
@ -25,7 +26,7 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('username, password, expected_exception', [
@ -45,8 +46,12 @@ def test_validated_keystone(username, password, expected_exception, app):
config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
config['KEYSTONE_ADMIN_TENANT'] = 'admintenant'
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)
else:
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)

18
util/config/validators/test/test_validate_ldap.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ldap import LDAPValidator
from util.morecollections import AttrDict
@ -7,13 +8,15 @@ from util.morecollections import AttrDict
from test.test_ldap import mock_ldap
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'LDAP'}),
@ -21,7 +24,8 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('uri', [
@ -39,7 +43,8 @@ def test_invalid_uri(uri, app):
config['LDAP_URI'] = uri
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(config, None, None)
config = ValidatorContext(config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('username, password, expected_exception', [
@ -56,10 +61,13 @@ def test_validated_ldap(username, password, expected_exception, app):
config['LDAP_ADMIN_PASSWD'] = 'password'
config['LDAP_USER_RDN'] = ['ou=employees']
unvalidated_config = ValidatorContext(config, user_password=password, config_provider=config_provider)
unvalidated_config.user = AttrDict(dict(username=username))
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)
else:
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)

11
util/config/validators/test/test_validate_oidc.py
View file

@ -3,7 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from oauth.oidc import OIDC_WELLKNOWN
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_oidc import OIDCLoginValidator
@ -19,7 +21,7 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, app):
validator = OIDCLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_oidc_login(app):
url_hit = [False]
@ -33,13 +35,16 @@ def test_validate_oidc_login(app):
with HTTMock(handler):
validator = OIDCLoginValidator()
validator.validate({
unvalidated_config = ValidatorContext({
'SOMETHING_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
'OIDC_SERVER': 'http://someserver',
'DEBUGGING': True, # Allows for HTTP.
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

12
util/config/validators/test/test_validate_redis.py
View file

@ -5,10 +5,13 @@ from mock import patch
from mockredis import mock_strict_redis_client
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_redis import RedisValidator
from test.fixtures import *
from util.morecollections import AttrDict
@pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
({}, None, None, False, ConfigValidationException),
@ -19,8 +22,13 @@ from test.fixtures import *
def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
validator = RedisValidator()
unvalidated_config = ValidatorContext(unvalidated_config)
unvalidated_config.user = AttrDict(dict(username=user))
unvalidated_config.user_password = user_password
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

19
util/config/validators/test/test_validate_secscan.py
View file

@ -1,6 +1,8 @@
import pytest
from util.config.validators import ConfigValidationException
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators.validate_secscan import SecurityScannerValidator
from util.secscan.fake import fake_security_scanner
@ -10,7 +12,12 @@ from test.fixtures import *
({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
])
def test_validate_noop(unvalidated_config, app):
SecurityScannerValidator.validate(unvalidated_config, None, None)
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
SecurityScannerValidator.validate(unvalidated_config)
@pytest.mark.parametrize('unvalidated_config, expected_error', [
@ -29,9 +36,13 @@ def test_validate_noop(unvalidated_config, app):
}, None),
])
def test_validate(unvalidated_config, expected_error, app):
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
with fake_security_scanner(hostname='fakesecurityscanner'):
if expected_error is not None:
with pytest.raises(expected_error):
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)
else:
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)

5
util/config/validators/test/test_validate_signer.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_signer import SignerValidator
@ -14,6 +15,6 @@ def test_validate_signer(unvalidated_config, expected, app):
validator = SignerValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

10
util/config/validators/test/test_validate_ssl.py
View file

@ -3,11 +3,13 @@ import pytest
from mock import patch
from tempfile import NamedTemporaryFile
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
from test.test_ssl_util import generate_test_cert
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
@ -16,7 +18,7 @@ from test.fixtures import *
])
def test_skip_validate_ssl(unvalidated_config, app):
validator = SSLValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('cert, expected_error, error_message', [
@ -54,11 +56,13 @@ def test_validate_ssl(cert, expected_error, error_message, app):
with patch('app.config_provider.volume_file_exists', return_true):
with patch('app.config_provider.get_volume_file', get_volume_file):
validator = SSLValidator()
config = ValidatorContext(config)
config.config_provider = config_provider
if expected_error is not None:
with pytest.raises(expected_error) as ipe:
validator.validate(config, None, None)
validator.validate(config)
assert ipe.value.message == error_message
else:
validator.validate(config, None, None)
validator.validate(config)

9
util/config/validators/test/test_validate_storage.py
View file

@ -1,6 +1,7 @@
import moto
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_storage import StorageValidator
@ -16,15 +17,15 @@ def test_validate_storage(unvalidated_config, expected, app):
validator = StorageValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_s3_storage(app):
validator = StorageValidator()
with moto.mock_s3():
with pytest.raises(ConfigValidationException) as ipe:
validator.validate({
validator.validate(ValidatorContext({
'DISTRIBUTED_STORAGE_CONFIG': {
'default': ('S3Storage', {
's3_access_key': 'invalid',
@ -33,6 +34,6 @@ def test_validate_s3_storage(app):
'storage_path': ''
}),
}
}, None, None)
}))
assert ipe.value.message == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'

7
util/config/validators/test/test_validate_timemachine.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_timemachine import TimeMachineValidator
@ -7,7 +8,7 @@ from util.config.validators.validate_timemachine import TimeMachineValidator
({}),
])
def test_validate_noop(unvalidated_config):
TimeMachineValidator.validate(unvalidated_config, None, None)
TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
from test.fixtures import *
@ -25,7 +26,7 @@ def test_validate(default_exp, options, expected_exception, app):
if expected_exception is not None:
with pytest.raises(ConfigValidationException) as cve:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))
assert str(cve.value) == str(expected_exception)
else:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))

12
util/config/validators/test/test_validate_torrent.py
View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_torrent import BittorrentValidator
@ -23,8 +25,14 @@ def test_validate_torrent(unvalidated_config, expected, app):
validator = BittorrentValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert not announcer_hit[0]
else:
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert announcer_hit[0]

7
util/config/validators/validate_access.py
View file

@ -1,4 +1,3 @@
from app import app
from util.config.validators import BaseValidator, ConfigValidationException
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService
@ -7,13 +6,15 @@ class AccessSettingsValidator(BaseValidator):
name = "access"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
if not config.get('FEATURE_DIRECT_LOGIN', True):
# Make sure we have at least one OIDC enabled.
github_login = config.get('FEATURE_GITHUB_LOGIN', False)
google_login = config.get('FEATURE_GOOGLE_LOGIN', False)
client = app.config['HTTPCLIENT']
login_manager = OAuthLoginManager(config, client=client)
custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)]

4
util/config/validators/validate_actionlog_archiving.py
View file

@ -4,7 +4,9 @@ class ActionLogArchivingValidator(BaseValidator):
name = "actionlogarchiving"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
""" Validates the action log archiving configuration. """
if not config.get('FEATURE_ACTION_LOG_ROTATION', False):
return

4
util/config/validators/validate_apptokenauth.py
View file

@ -4,7 +4,9 @@ class AppTokenAuthValidator(BaseValidator):
name = "apptoken-auth"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken':
return

7
util/config/validators/validate_bitbucket_trigger.py
View file

@ -1,14 +1,15 @@
from bitbucket import BitBucket
from app import get_app_url
from util.config.validators import BaseValidator, ConfigValidationException
class BitbucketTriggerValidator(BaseValidator):
name = "bitbucket-trigger"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the config for BitBucket. """
config = validator_context.config
trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
if not trigger_config:
raise ConfigValidationException('Missing client ID and client secret')
@ -21,7 +22,7 @@ class BitbucketTriggerValidator(BaseValidator):
key = trigger_config['CONSUMER_KEY']
secret = trigger_config['CONSUMER_SECRET']
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url())
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (validator_context.url_scheme_and_hostname.get_url())
bitbucket_client = BitBucket(key, secret, callback_url)
(result, _, _) = bitbucket_client.get_authorization_url()

4
util/config/validators/validate_database.py
View file

@ -7,8 +7,10 @@ class DatabaseValidator(BaseValidator):
name = "database"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates connecting to the database. """
config = validator_context.config
try:
validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
except OperationalError as ex:

12
util/config/validators/validate_email.py
View file

@ -1,16 +1,20 @@
from flask import Flask
from flask_mail import Mail, Message
from app import app
from util.config.validators import BaseValidator
class EmailValidator(BaseValidator):
name = "mail"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates sending email. """
with app.app_context():
config = validator_context.config
user = validator_context.user
app_context = validator_context.context
registry_title = validator_context.registry_title
with app_context():
test_app = Flask("mail-test-app")
test_app.config.update(config)
test_app.config.update({
@ -19,7 +23,7 @@ class EmailValidator(BaseValidator):
})
test_mail = Mail(test_app)
test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'],
test_msg = Message("Test e-mail from %s" % registry_title,
sender=config.get('MAIL_DEFAULT_SENDER'))
test_msg.add_recipient(user.email)
test_mail.send(test_msg)

10
util/config/validators/validate_github.py
View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.github import GithubOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -7,8 +6,12 @@ class BaseGitHubValidator(BaseValidator):
config_key = None
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a Github service. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
github_config = config.get(cls.config_key)
if not github_config:
raise ConfigValidationException('Missing GitHub client id and client secret')
@ -30,9 +33,8 @@ class BaseGitHubValidator(BaseValidator):
raise ConfigValidationException('Organization restriction must have at least one allowed ' +
'organization')
client = app.config['HTTPCLIENT']
oauth = GithubOAuthService(config, cls.config_key)
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

10
util/config/validators/validate_gitlab_trigger.py
View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.gitlab import GitLabOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +5,12 @@ class GitLabTriggerValidator(BaseValidator):
name = "gitlab-trigger"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a GitLab service. """
config = validator_context.config
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
client = validator_context.http_client
github_config = config.get('GITLAB_TRIGGER_CONFIG')
if not github_config:
raise ConfigValidationException('Missing GitLab client id and client secret')
@ -23,8 +26,7 @@ class GitLabTriggerValidator(BaseValidator):
if not github_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
client = app.config['HTTPCLIENT']
oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG')
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

10
util/config/validators/validate_google_login.py
View file

@ -1,4 +1,3 @@
from app import app
from oauth.services.google import GoogleOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +5,12 @@ class GoogleLoginValidator(BaseValidator):
name = "google-login"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the Google Login client ID and secret. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
if not google_login_config:
raise ConfigValidationException('Missing client ID and client secret')
@ -18,8 +21,7 @@ class GoogleLoginValidator(BaseValidator):
if not google_login_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
client = app.config['HTTPCLIENT']
oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
result = oauth.validate_client_id_and_secret(client, app.config)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

23
util/config/validators/validate_jwt.py
View file

@ -1,4 +1,4 @@
from app import app, OVERRIDE_CONFIG_DIRECTORY
import os
from data.users.externaljwt import ExternalJWTAuthN
from util.config.validators import BaseValidator, ConfigValidationException
@ -6,8 +6,15 @@ class JWTAuthValidator(BaseValidator):
name = "jwt"
@classmethod
def validate(cls, config, user, user_password, public_key_path=None):
def validate(cls, validator_context, public_key_path=None):
""" Validates the JWT authentication system. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
http_client = validator_context.http_client
jwt_auth_max = validator_context.jwt_auth_max
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
return
@ -23,12 +30,18 @@ class JWTAuthValidator(BaseValidator):
if not issuer:
raise ConfigValidationException('Missing JWT Issuer ID')
# TODO(jschorr): fix this
return
override_config_directory = os.path.join(config_provider.get_config_root(), '../stack/')
# Try to instatiate the JWT authentication mechanism. This will raise an exception if
# the key cannot be found.
users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
OVERRIDE_CONFIG_DIRECTORY,
app.config['HTTPCLIENT'],
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
override_config_directory,
http_client,
jwt_auth_max,
public_key_path=public_key_path,
requires_email=config.get('FEATURE_MAILING', True))

6
util/config/validators/validate_keystone.py
View file

@ -5,8 +5,12 @@ class KeystoneValidator(BaseValidator):
name = "keystone"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the Keystone authentication system. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
return

11
util/config/validators/validate_ldap.py
View file

@ -2,24 +2,27 @@ import os
import ldap
import subprocess
from app import app, config_provider
from data.users import LDAP_CERT_FILENAME
from data.users.externalldap import LDAPConnection, LDAPUsers
from util.config.validators import BaseValidator, ConfigValidationException
from _init import CONF_DIR
class LDAPValidator(BaseValidator):
name = "ldap"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the LDAP connection. """
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
return
# If there is a custom LDAP certificate, then reinstall the certificates for the container.
if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
subprocess.check_call([os.path.join(CONF_DIR, 'init/certs_install.sh')])
subprocess.check_call([os.path.join(config_provider.get_config_root(), '../init/certs_install.sh')])
# Note: raises ldap.INVALID_CREDENTIALS on failure
admin_dn = config.get('LDAP_ADMIN_DN')

7
util/config/validators/validate_oidc.py
View file

@ -1,4 +1,3 @@
from app import app
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService, DiscoveryFailureException
from util.config.validators import BaseValidator, ConfigValidationException
@ -7,8 +6,10 @@ class OIDCLoginValidator(BaseValidator):
name = "oidc-login"
@classmethod
def validate(cls, config, user, user_password):
client = app.config['HTTPCLIENT']
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
login_manager = OAuthLoginManager(config, client=client)
for service in login_manager.services:
if not isinstance(service, OIDCLoginService):

4
util/config/validators/validate_redis.py
View file

@ -6,8 +6,10 @@ class RedisValidator(BaseValidator):
name = "redis"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates connecting to redis. """
config = validator_context.config
redis_config = config.get('BUILDLOGS_REDIS', {})
if not 'host' in redis_config:
raise ConfigValidationException('Missing redis hostname')

18
util/config/validators/validate_secscan.py
View file

@ -1,6 +1,5 @@
import time
from app import app
from boot import setup_jwt_proxy
from util.secscan.api import SecurityScannerAPI
from util.config.validators import BaseValidator, ConfigValidationException
@ -9,15 +8,22 @@ class SecurityScannerValidator(BaseValidator):
name = "security-scanner"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the configuration for talking to a Quay Security Scanner. """
if not config.get('FEATURE_SECURITY_SCANNER', False):
config = validator_context.config
client = validator_context.http_client
feature_sec_scanner = validator_context.feature_sec_scanner
is_testing = validator_context.is_testing
server_hostname = validator_context.url_scheme_and_hostname.hostname
uri_creator = validator_context.uri_creator
if not feature_sec_scanner:
return
client = app.config['HTTPCLIENT']
api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)
api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
if not config.get('TESTING', False):
if not is_testing:
# Generate a temporary Quay key to use for signing the outgoing requests.
setup_jwt_proxy()

6
util/config/validators/validate_signer.py
View file

@ -1,6 +1,5 @@
from StringIO import StringIO
from app import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.signing import SIGNING_ENGINES
@ -8,8 +7,11 @@ class SignerValidator(BaseValidator):
name = "signer"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the GPG public+private key pair used for signing converted ACIs. """
config = validator_context.config
config_provider = validator_context.config_provider
if config.get('SIGNING_ENGINE') is None:
return

5
util/config/validators/validate_ssl.py
View file

@ -1,4 +1,3 @@
from app import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
@ -8,8 +7,10 @@ class SSLValidator(BaseValidator):
name = "ssl"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the SSL configuration (if enabled). """
config = validator_context.config
config_provider = validator_context.config_provider
# Skip if non-SSL.
if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':

15
util/config/validators/validate_storage.py
View file

@ -1,16 +1,21 @@
from app import app, ip_resolver, config_provider
from storage import get_storage_driver
from util.config.validators import BaseValidator, ConfigValidationException
class StorageValidator(BaseValidator):
name = "registry-storage"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates registry storage. """
config = validator_context.config
client = validator_context.http_client
ip_resolver = validator_context.ip_resolver
config_provider = validator_context.config_provider
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
providers = _get_storage_providers(config).items()
providers = _get_storage_providers(config, ip_resolver, config_provider).items()
if not providers:
raise ConfigValidationException('Storage configuration required')
@ -21,7 +26,7 @@ class StorageValidator(BaseValidator):
'with storage replication')
# Run validation on the driver.
driver.validate(app.config['HTTPCLIENT'])
driver.validate(client)
# Run setup on the driver if the read/write succeeded.
driver.setup()
@ -30,7 +35,7 @@ class StorageValidator(BaseValidator):
raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
def _get_storage_providers(config):
def _get_storage_providers(config, ip_resolver, config_provider):
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
drivers = {}

4
util/config/validators/validate_timemachine.py
View file

@ -9,7 +9,9 @@ class TimeMachineValidator(BaseValidator):
name = "time-machine"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
config = validator_context.config
if not 'DEFAULT_TAG_EXPIRATION' in config:
# Old style config
return

8
util/config/validators/validate_torrent.py
View file

@ -2,7 +2,6 @@ import logging
from hashlib import sha1
from app import app
from util.config.validators import BaseValidator, ConfigValidationException
from util.registry.torrent import jwt_from_infohash
@ -12,15 +11,16 @@ class BittorrentValidator(BaseValidator):
name = "bittorrent"
@classmethod
def validate(cls, config, user, user_password):
def validate(cls, validator_context):
""" Validates the configuration for using BitTorrent for downloads. """
config = validator_context.config
client = validator_context.http_client
announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
if not announce_url:
raise ConfigValidationException('Missing announce URL')
# Ensure that the tracker is reachable and accepts requests signed with a registry key.
client = app.config['HTTPCLIENT']
params = {
'info_hash': sha1('test').digest(),
'peer_id': '-QUAY00-6wfG2wk6wWLc',

30
util/secscan/api.py
View file

@ -7,16 +7,12 @@ from urlparse import urljoin
import requests
from flask import url_for
from data.database import CloseForLongOperation
from data import model
from data.model.storage import get_storage_locations
from util import get_app_url, slash_join
from util.abchelpers import nooper
from util.failover import failover, FailoverException
from util.secscan.validator import SecurityConfigValidator
from util.security.instancekeys import InstanceKeys
from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
from _init import CONF_DIR
@ -70,16 +66,16 @@ def compute_layer_id(layer):
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (usually Clair). """
def __init__(self, app, config, storage, client=None, skip_validation=False):
def __init__(self, config, storage, server_hostname=None, client=None, skip_validation=False, uri_creator=None, instance_keys=None):
feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False)
has_valid_config = skip_validation
if not skip_validation and feature_enabled:
config_validator = SecurityConfigValidator(config)
config_validator = SecurityConfigValidator(feature_enabled, config.get('SECURITY_SCANNER_ENDPOINT'))
has_valid_config = config_validator.valid()
if feature_enabled and has_valid_config:
self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client)
self.state = ImplementedSecurityScannerAPI(config, storage, server_hostname, client=client, uri_creator=uri_creator, instance_keys=instance_keys)
else:
self.state = NoopSecurityScannerAPI()
@ -150,20 +146,25 @@ class NoopSecurityScannerAPI(SecurityScannerAPIInterface):
class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
""" Helper class for talking to the Security Scan service (Clair). """
def __init__(self, app, config, storage, client=None):
self._app = app
# TODO(sam) refactor this to not take an app config, and instead just the things it needs as a config object
def __init__(self, config, storage, server_hostname, client=None, uri_creator=None, instance_keys=None):
self._config = config
self._instance_keys = InstanceKeys(app)
self._client = client or config['HTTPCLIENT']
self._instance_keys = instance_keys
self._client = client
self._storage = storage
self._server_hostname = server_hostname
self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
self._uri_creator = uri_creator
def _get_image_url_and_auth(self, image):
""" Returns a tuple of the url and the auth header value that must be used
to fetch the layer data itself. If the image can't be addressed, we return
None.
"""
if self._instance_keys is None:
raise Exception('No Instance keys provided to Security Scanner API')
path = model.storage.get_layer_path(image.storage)
locations = self._default_storage_locations
@ -183,7 +184,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
repository_and_namespace = '/'.join([namespace_name, repo_name])
# Generate the JWT which will authorize this
audience = self._app.config['SERVER_HOSTNAME']
audience = self._server_hostname
context, subject = build_context_and_subject()
access = [{
'type': 'repository',
@ -195,10 +196,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
auth_header = 'Bearer ' + auth_token
with self._app.test_request_context('/'):
relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
digest=image.storage.content_checksum)
uri = urljoin(get_app_url(self._config), relative_layer_url)
uri = self._uri_creator(repository_and_namespace, image.storage.content_checksum)
return uri, auth_header

22
util/secscan/secscan_util.py Normal file
View file

@ -0,0 +1,22 @@
from urlparse import urljoin
from flask import url_for
def get_blob_download_uri_getter(context, url_scheme_and_hostname):
"""
Returns a function with context to later generate the uri for a download blob
:param context: Flask RequestContext
:param url_scheme_and_hostname: URLSchemeAndHostname class instance
:return: function (repository_and_namespace, checksum) -> uri
"""
def create_uri(repository_and_namespace, checksum):
"""
Creates a uri for a download blob from a repository, namespace, and checksum from earlier context
"""
with context:
relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
digest=checksum)
return urljoin(url_scheme_and_hostname.get_url(), relative_layer_url)
return create_uri

19
util/secscan/test/test_secscan_util.py Normal file
View file

@ -0,0 +1,19 @@
import pytest
from app import app
from util.config import URLSchemeAndHostname
from util.secscan.secscan_util import get_blob_download_uri_getter
from test.fixtures import *
@pytest.mark.parametrize('url_scheme_and_hostname, repo_namespace, checksum, expected_value,', [
(URLSchemeAndHostname('http', 'localhost:5000'),
'devtable/simple', 'tarsum+sha256:123',
'http://localhost:5000/v2/devtable/simple/blobs/tarsum+sha256:123'),
])
def test_blob_download_uri_getter(app, url_scheme_and_hostname,
repo_namespace, checksum,
expected_value):
blob_uri_getter = get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname)
assert blob_uri_getter(repo_namespace, checksum) == expected_value

15
util/secscan/validator.py
View file

@ -1,28 +1,27 @@
import logging
import features
logger = logging.getLogger(__name__)
class SecurityConfigValidator(object):
""" Helper class for validating the security scanner configuration. """
def __init__(self, config):
if not features.SECURITY_SCANNER:
def __init__(self, feature_sec_scan, sec_scan_endpoint):
if not feature_sec_scan:
return
self._config = config
self._feature_sec_scan = feature_sec_scan
self._sec_scan_endpoint = sec_scan_endpoint
def valid(self):
if not features.SECURITY_SCANNER:
if not self._feature_sec_scan:
return False
if self._config.get('SECURITY_SCANNER_ENDPOINT') is None:
if self._sec_scan_endpoint is None:
logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
return False
endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT')
endpoint = self._sec_scan_endpoint
if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
return False