diff --git a/app.py b/app.py index 24071c5be..2faaf58a6 100644 --- a/app.py +++ b/app.py @@ -35,8 +35,9 @@ from oauth.services.github import GithubOAuthService from oauth.services.gitlab import GitLabOAuthService from oauth.loginmanager import OAuthLoginManager from storage import Storage +from util.config import URLSchemeAndHostname from util.log import filter_logs -from util import get_app_url +from util import get_app_url, create_uri_func_from_context from util.ipresolver import IPResolver from util.saas.analytics import Analytics from util.saas.useranalytics import UserAnalytics @@ -50,7 +51,7 @@ from util.metrics.prometheus import PrometheusPlugin from util.saas.cloudwatch import start_cloudwatch_sender from util.secscan.api import SecurityScannerAPI from util.tufmetadata.api import TUFMetadataAPI -from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config +from util.security.instancekeys import InstanceKeys from util.security.signing import Signer @@ -182,7 +183,7 @@ mail = Mail(app) prometheus = PrometheusPlugin(app) metric_queue = MetricQueue(prometheus) chunk_cleanup_queue = WorkQueue(app.config['CHUNK_CLEANUP_QUEUE_NAME'], tf, metric_queue=metric_queue) -instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config)) +instance_keys = InstanceKeys(app) ip_resolver = IPResolver(app) storage = Storage(app, metric_queue, chunk_cleanup_queue, instance_keys, config_provider, ip_resolver) userfiles = Userfiles(app, storage) @@ -196,7 +197,7 @@ authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECT userevents = UserEventsBuilderModule(app) superusers = SuperUserManager(app) signer = Signer(app, config_provider) -instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config)) +instance_keys = InstanceKeys(app) label_validator = LabelValidator(app) build_canceller = BuildCanceller(app) @@ -228,7 +229,11 @@ namespace_gc_queue = WorkQueue(app.config['NAMESPACE_GC_QUEUE_NAME'], tf, has_na all_queues = [image_replication_queue, dockerfile_build_queue, notification_queue, secscan_notification_queue, chunk_cleanup_queue, namespace_gc_queue] -secscan_api = SecurityScannerAPI(app, app.config, storage) +url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) +secscan_api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'], + uri_creator=create_uri_func_from_context(app.test_request_context('/'), url_scheme_and_hostname), + instance_keys=instance_keys) + tuf_metadata_api = TUFMetadataAPI(app, app.config) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 12b4918bb..0f88bd234 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -405,7 +405,10 @@ class SuperUserConfigValidate(ApiResource): # this is also safe since this method does not access any information not given in the request. if not config_provider.config_exists() or SuperUserPermission().can(): config = request.get_json()['config'] - validator_context = ValidatorContext.from_app(config, request.get_json().get('password', ''), app, ip_resolver) + validator_context = ValidatorContext.from_app(config, request.get_json().get('password', ''), app, + ip_resolver=ip_resolver, + config_provider=config_provider) + return validate_service_for_config(service, validator_context) abort(403) diff --git a/endpoints/githubtrigger.py b/endpoints/githubtrigger.py index 3a7e7ddd0..3b4b21f0a 100644 --- a/endpoints/githubtrigger.py +++ b/endpoints/githubtrigger.py @@ -10,7 +10,6 @@ from auth.decorators import require_session_login from auth.permissions import AdministerRepositoryPermission from data import model from endpoints.decorators import route_show_if, parse_repository_name -from util.config import URLSchemeAndHostname from util.http import abort @@ -27,7 +26,6 @@ def attach_github_build_trigger(namespace_name, repo_name): permission = AdministerRepositoryPermission(namespace_name, repo_name) if permission.can(): code = request.args.get('code') - # url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) token = github_trigger.exchange_code_for_token(app.config, client, code) repo = model.repository.get_repository(namespace_name, repo_name) if not repo: diff --git a/endpoints/gitlabtrigger.py b/endpoints/gitlabtrigger.py index d8e27332b..4d97caffe 100644 --- a/endpoints/gitlabtrigger.py +++ b/endpoints/gitlabtrigger.py @@ -10,7 +10,6 @@ from auth.decorators import require_session_login from auth.permissions import AdministerRepositoryPermission from data import model from endpoints.decorators import route_show_if -from util.config import URLSchemeAndHostname from util.http import abort @@ -35,7 +34,6 @@ def attach_gitlab_build_trigger(): permission = AdministerRepositoryPermission(namespace, repository) if permission.can(): code = request.args.get('code') - # url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) token = gitlab_trigger.exchange_code_for_token(app.config, client, code, redirect_suffix='/trigger') if not token: diff --git a/oauth/login.py b/oauth/login.py index 531a55e6c..55c94be69 100644 --- a/oauth/login.py +++ b/oauth/login.py @@ -6,7 +6,6 @@ from six import add_metaclass import features from oauth.base import OAuthService, OAuthExchangeCodeException, OAuthGetUserInfoException -from util.config import URLSchemeAndHostname logger = logging.getLogger(__name__) @@ -65,7 +64,6 @@ class OAuthLoginService(OAuthService): # Retrieve the token for the OAuth code. try: - # url_scheme_and_hostname = URLSchemeAndHostname(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME']) token = self.exchange_code_for_token(app_config, http_client, code, redirect_suffix=redirect_suffix, form_encode=self.requires_form_encoding()) diff --git a/oauth/services/github.py b/oauth/services/github.py index 6bc9350ff..25f0d65d7 100644 --- a/oauth/services/github.py +++ b/oauth/services/github.py @@ -75,7 +75,8 @@ class GithubOAuthService(OAuthLoginService): def orgs_endpoint(self): return slash_join(self._api_endpoint(), 'user/orgs') - def validate_client_id_and_secret(self, http_client, app_config): + # TODO(sam): refactor the base method to not take app config + def validate_client_id_and_secret(self, http_client): # First: Verify that the github endpoint is actually Github by checking for the # X-GitHub-Request-Id here. api_endpoint = self._api_endpoint() diff --git a/oauth/services/google.py b/oauth/services/google.py index a22964bb6..892ac03b8 100644 --- a/oauth/services/google.py +++ b/oauth/services/google.py @@ -41,7 +41,8 @@ class GoogleOAuthService(OAuthLoginService): def requires_form_encoding(self): return True - def validate_client_id_and_secret(self, http_client, app_config): + # TODO(sam): this signature does not match its parent class. refactor the base method to take the namedtuple URLSchemeAndHostname + def validate_client_id_and_secret(self, http_client): # To verify the Google client ID and secret, we hit the # https://www.googleapis.com/oauth2/v3/token endpoint with an invalid request. If the client # ID or secret are invalid, we get returned a 403 Unauthorized. Otherwise, we get returned diff --git a/test/test_secscan.py b/test/test_secscan.py index 8ba559d95..62a70fd8f 100644 --- a/test/test_secscan.py +++ b/test/test_secscan.py @@ -8,11 +8,14 @@ from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION from endpoints.v2 import v2_bp from initdb import setup_database_for_testing, finished_database_for_testing from notifications.notificationevent import VulnerabilityFoundEvent +from util import create_uri_func_from_context +from util.config import URLSchemeAndHostname from util.morecollections import AttrDict from util.secscan.api import SecurityScannerAPI, APIRequestFailure from util.secscan.analyzer import LayerAnalyzer from util.secscan.fake import fake_security_scanner from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult +from util.security.instancekeys import InstanceKeys from workers.security_notification_worker import SecurityNotificationWorker @@ -42,7 +45,13 @@ class TestSecurityScanner(unittest.TestCase): self.ctx = app.test_request_context() self.ctx.__enter__() - self.api = SecurityScannerAPI(app, app.config, storage) + + url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) + instance_keys = InstanceKeys(app) + self.api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'], + uri_creator=create_uri_func_from_context(app.test_request_context('/'), + url_scheme_and_hostname), + instance_keys=instance_keys) def tearDown(self): storage.remove(['local_us'], 'supports_direct_download') diff --git a/test/test_storageproxy.py b/test/test_storageproxy.py index 3078018dd..174c3c854 100644 --- a/test/test_storageproxy.py +++ b/test/test_storageproxy.py @@ -7,7 +7,7 @@ from flask_testing import LiveServerTestCase from initdb import setup_database_for_testing, finished_database_for_testing from storage import Storage -from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config +from util.security.instancekeys import InstanceKeys _PORT_NUMBER = 5001 @@ -42,7 +42,7 @@ class TestStorageProxy(LiveServerTestCase): 'test': ['FakeStorage', {}], } - instance_keys = InstanceKeys(instance_keys_context_from_app_config(self.test_app.config)) + instance_keys = InstanceKeys(self.test_app) self.storage = Storage(self.test_app, instance_keys=instance_keys) self.test_app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = ['test'] return self.test_app diff --git a/util/__init__.py b/util/__init__.py index 53a25fb85..41511854f 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -1,3 +1,6 @@ +from flask import url_for +from urlparse import urljoin + def get_app_url(config): """ Returns the application's URL, based on the given config. """ return '%s://%s' % (config['PREFERRED_URL_SCHEME'], config['SERVER_HOSTNAME']) @@ -19,3 +22,13 @@ def slash_join(*args): args = [rmslash(path) for path in args] return '/'.join(args) + +def create_uri_func_from_context(context, url_scheme_and_hostname): + def create_uri(repository_and_namespace, checksum): + with context: + relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace, + digest=checksum) + return urljoin(get_app_url_from_scheme_hostname(url_scheme_and_hostname), relative_layer_url) + + return create_uri + diff --git a/util/config/provider/basefileprovider.py b/util/config/provider/basefileprovider.py index 7d0572351..14fcdebb0 100644 --- a/util/config/provider/basefileprovider.py +++ b/util/config/provider/basefileprovider.py @@ -68,3 +68,7 @@ class BaseFileProvider(BaseProvider): return True return False + + def get_config_root(self): + return self.config_volume + diff --git a/util/config/provider/baseprovider.py b/util/config/provider/baseprovider.py index 052b32e76..02d693cf0 100644 --- a/util/config/provider/baseprovider.py +++ b/util/config/provider/baseprovider.py @@ -123,3 +123,8 @@ class BaseProvider(object): def get_volume_path(self, directory, filename): """ Helper for constructing relative file paths, which may differ between providers. For example, kubernetes can't have subfolders in configmaps """ + + @abstractmethod + def get_config_root(self): + """ Returns the config root directory. """ + diff --git a/util/config/provider/testprovider.py b/util/config/provider/testprovider.py index 55158444b..e3df1478e 100644 --- a/util/config/provider/testprovider.py +++ b/util/config/provider/testprovider.py @@ -10,6 +10,11 @@ REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', ' class TestConfigProvider(BaseProvider): """ Implementation of the config provider for testing. Everything is kept in-memory instead on the real file system. """ + + def get_config_root(self): + raise Exception('Test Config does not have a config root') + # return '' + def __init__(self): self.clear() diff --git a/util/config/validator.py b/util/config/validator.py index 3e6aaf787..26be2da06 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -2,6 +2,7 @@ import logging from auth.auth_context import get_authenticated_user from data.users import LDAP_CERT_FILENAME +from util import create_uri_func_from_context from util.config import URLSchemeAndHostname from util.config.validators.validate_database import DatabaseValidator @@ -99,22 +100,40 @@ class ValidatorContext(object): """ Context to run validators in, with any additional runtime configuration they need """ def __init__(self, config, user_password=None, http_client=None, context=None, - url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, ip_resolver=None): + url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, + ip_resolver=None, feature_sec_scanner=False, is_testing=False, + uri_creator=None, config_provider=None): self.config = config self.user = get_authenticated_user() self.user_password = user_password self.http_client = http_client self.context = context - self.scheme_and_hostname = url_scheme_and_hostname + self.url_scheme_and_hostname = url_scheme_and_hostname self.jwt_auth_max = jwt_auth_max self.registry_title = registry_title self.ip_resolver = ip_resolver + self.feature_sec_scanner = feature_sec_scanner + self.is_testing = is_testing + self.uri_creator = uri_creator + self.config_provider = config_provider @classmethod - def from_app(cls, config, user_password, app, ip_resolver): - cls(config, user_password, app.config['HTTP_CLIENT'], app.app_context, - URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']), - app.config.get('JWT_AUTH_MAX_FRESH_S', 300), app.config['REGISTRY_TITLE'], ip_resolver) + def from_app(cls, config, user_password, app, ip_resolver, client=None, config_provider=None): + url_scheme = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) + + cls(config, + user_password, + client or app.config['HTTPCLIENT'], + app.app_context, + url_scheme, + app.config.get('JWT_AUTH_MAX_FRESH_S', 300), + app.config['REGISTRY_TITLE'], + ip_resolver, + app.config.get('FEATURE_SECURITY_SCANNER', False), + app.config.get('TESTING', False), + create_uri_func_from_context(app.test_request_context('/'), url_scheme), + config_provider) + diff --git a/util/config/validators/test/test_validate_access.py b/util/config/validators/test/test_validate_access.py index 9008767eb..54f44649b 100644 --- a/util/config/validators/test/test_validate_access.py +++ b/util/config/validators/test/test_validate_access.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_access import AccessSettingsValidator @@ -17,6 +18,6 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, expected_excepti if expected_exception is not None: with pytest.raises(expected_exception): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) else: - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) diff --git a/util/config/validators/test/test_validate_actionlog_archiving.py b/util/config/validators/test/test_validate_actionlog_archiving.py index c14555441..e3641e49c 100644 --- a/util/config/validators/test/test_validate_actionlog_archiving.py +++ b/util/config/validators/test/test_validate_actionlog_archiving.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator @@ -12,7 +13,7 @@ from test.fixtures import * ]) def test_skip_validate_actionlog(unvalidated_config, app): validator = ActionLogArchivingValidator() - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('config, expected_error', [ @@ -33,19 +34,19 @@ def test_invalid_config(config, expected_error, app): validator = ActionLogArchivingValidator() with pytest.raises(ConfigValidationException) as ipe: - validator.validate(config, None, None) + validator.validate(ValidatorContext(config)) assert ipe.value.message == expected_error def test_valid_config(app): - config = { + config = ValidatorContext({ 'FEATURE_ACTION_LOG_ROTATION': True, 'ACTION_LOG_ARCHIVE_PATH': 'somepath', 'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation', 'DISTRIBUTED_STORAGE_CONFIG': { 'somelocation': {}, }, - } + }) validator = ActionLogArchivingValidator() - validator.validate(config, None, None) + validator.validate(config) diff --git a/util/config/validators/test/test_validate_apptokenauth.py b/util/config/validators/test/test_validate_apptokenauth.py index 61ccd3570..87227f702 100644 --- a/util/config/validators/test/test_validate_apptokenauth.py +++ b/util/config/validators/test/test_validate_apptokenauth.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_apptokenauth import AppTokenAuthValidator @@ -15,15 +16,15 @@ def test_validate_invalid_auth_config(unvalidated_config, app): validator = AppTokenAuthValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_auth(app): - config = { + config = ValidatorContext({ 'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': True, 'FEATURE_DIRECT_LOGIN': False, - } + }) validator = AppTokenAuthValidator() - validator.validate(config, None, None) + validator.validate(config) diff --git a/util/config/validators/test/test_validate_bitbucket_trigger.py b/util/config/validators/test/test_validate_bitbucket_trigger.py index a5b5b6738..59b2a748d 100644 --- a/util/config/validators/test/test_validate_bitbucket_trigger.py +++ b/util/config/validators/test/test_validate_bitbucket_trigger.py @@ -2,22 +2,24 @@ import pytest from httmock import urlmatch, HTTMock +from util.config import URLSchemeAndHostname +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator from test.fixtures import * @pytest.mark.parametrize('unvalidated_config', [ - ({}), - ({'BITBUCKET_TRIGGER_CONFIG': {}}), - ({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}}), - ({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}}), + (ValidatorContext({})), + (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})), + (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})), + (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})), ]) def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app): validator = BitbucketTriggerValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(unvalidated_config) def test_validate_bitbucket_trigger(app): url_hit = [False] @@ -32,11 +34,16 @@ def test_validate_bitbucket_trigger(app): with HTTMock(handler): validator = BitbucketTriggerValidator() - validator.validate({ + + unvalidated_config = ValidatorContext({ 'BITBUCKET_TRIGGER_CONFIG': { 'CONSUMER_KEY': 'foo', 'CONSUMER_SECRET': 'bar', }, - }, None, None) + }) + + unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000') + + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_database.py b/util/config/validators/test/test_validate_database.py index 20e1d97e5..68b21acd6 100644 --- a/util/config/validators/test/test_validate_database.py +++ b/util/config/validators/test/test_validate_database.py @@ -1,22 +1,23 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_database import DatabaseValidator from test.fixtures import * @pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [ - (None, None, None, TypeError), - ({}, None, None, KeyError), - ({'DB_URI': 'sqlite:///:memory:'}, None, None, None), - ({'DB_URI': 'invalid:///:memory:'}, None, None, KeyError), - ({'DB_NOTURI': 'sqlite:///:memory:'}, None, None, KeyError), + (ValidatorContext(None), None, None, TypeError), + (ValidatorContext({}), None, None, KeyError), + (ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None), + (ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError), + (ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError), ]) def test_validate_database(unvalidated_config, user, user_password, expected, app): validator = DatabaseValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) else: - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_github.py b/util/config/validators/test/test_validate_github.py index aad09870f..b0e919c22 100644 --- a/util/config/validators/test/test_validate_github.py +++ b/util/config/validators/test/test_validate_github.py @@ -2,6 +2,8 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator @@ -36,7 +38,7 @@ def test_validate_invalid_github_config(github_config, github_validator, app): with pytest.raises(ConfigValidationException): unvalidated_config = {} unvalidated_config[github_validator.config_key] = github_config - github_validator.validate(unvalidated_config, None, None) + github_validator.validate(ValidatorContext(unvalidated_config)) def test_validate_github(github_validator, app): url_hit = [False, False] @@ -52,13 +54,16 @@ def test_validate_github(github_validator, app): return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}} with HTTMock(app_handler, handler): - github_validator.validate({ + unvalidated_config = ValidatorContext({ github_validator.config_key: { 'GITHUB_ENDPOINT': 'http://somehost', 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', }, - }, None, None) + }) + + unvalidated_config.http_client = build_requests_session() + github_validator.validate(unvalidated_config) assert url_hit[0] assert url_hit[1] diff --git a/util/config/validators/test/test_validate_gitlab_trigger.py b/util/config/validators/test/test_validate_gitlab_trigger.py index 17b32764b..d374ade78 100644 --- a/util/config/validators/test/test_validate_gitlab_trigger.py +++ b/util/config/validators/test/test_validate_gitlab_trigger.py @@ -3,6 +3,9 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config import URLSchemeAndHostname +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator @@ -18,7 +21,7 @@ def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app): validator = GitLabTriggerValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_gitlab_enterprise_trigger(app): url_hit = [False] @@ -30,12 +33,16 @@ def test_validate_gitlab_enterprise_trigger(app): with HTTMock(handler): validator = GitLabTriggerValidator() - validator.validate({ + unvalidated_config = ValidatorContext({ 'GITLAB_TRIGGER_CONFIG': { 'GITLAB_ENDPOINT': 'http://somegitlab', 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', }, - }, None, None) + }) + unvalidated_config.http_client = build_requests_session() + + unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000') + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_google_login.py b/util/config/validators/test/test_validate_google_login.py index a41a51adb..ff823a7a7 100644 --- a/util/config/validators/test/test_validate_google_login.py +++ b/util/config/validators/test/test_validate_google_login.py @@ -2,6 +2,8 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_google_login import GoogleLoginValidator @@ -17,7 +19,7 @@ def test_validate_invalid_google_login_config(unvalidated_config, app): validator = GoogleLoginValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_google_login(app): url_hit = [False] @@ -29,11 +31,15 @@ def test_validate_google_login(app): validator = GoogleLoginValidator() with HTTMock(handler): - validator.validate({ + unvalidated_config = ValidatorContext({ 'GOOGLE_LOGIN_CONFIG': { 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', }, - }, None, None) + }) + + unvalidated_config.http_client = build_requests_session() + + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_jwt.py b/util/config/validators/test/test_validate_jwt.py index e535f7f9e..fec57a55f 100644 --- a/util/config/validators/test/test_validate_jwt.py +++ b/util/config/validators/test/test_validate_jwt.py @@ -1,5 +1,7 @@ import pytest +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_jwt import JWTAuthValidator from util.morecollections import AttrDict @@ -7,6 +9,7 @@ from util.morecollections import AttrDict from test.test_external_jwt_authn import fake_jwt from test.fixtures import * +from app import config_provider @pytest.mark.parametrize('unvalidated_config', [ @@ -14,7 +17,9 @@ from test.fixtures import * ({'AUTHENTICATION_TYPE': 'Database'}), ]) def test_validate_noop(unvalidated_config, app): - JWTAuthValidator.validate(unvalidated_config, None, None, app) + config = ValidatorContext(unvalidated_config) + config.config_provider = config_provider + JWTAuthValidator.validate(config) @pytest.mark.parametrize('unvalidated_config', [ @@ -24,7 +29,9 @@ def test_validate_noop(unvalidated_config, app): ]) def test_invalid_config(unvalidated_config, app): with pytest.raises(ConfigValidationException): - JWTAuthValidator.validate(unvalidated_config, None, None, app) + config = ValidatorContext(unvalidated_config) + config.config_provider = config_provider + JWTAuthValidator.validate(config) @pytest.mark.parametrize('username, password, expected_exception', [ @@ -42,10 +49,15 @@ def test_validated_jwt(username, password, expected_exception, app): config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url + unvalidated_config = ValidatorContext(config) + unvalidated_config.user = AttrDict(dict(username=username)) + unvalidated_config.user_password = password + unvalidated_config.config_provider = config_provider + + unvalidated_config.http_client = build_requests_session() + if expected_exception is not None: with pytest.raises(ConfigValidationException): - JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app, - public_key_path=jwt_auth.public_key_path) + JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path) else: - JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app, - public_key_path=jwt_auth.public_key_path) + JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path) diff --git a/util/config/validators/test/test_validate_keystone.py b/util/config/validators/test/test_validate_keystone.py index 304700b39..f6d8cf0c2 100644 --- a/util/config/validators/test/test_validate_keystone.py +++ b/util/config/validators/test/test_validate_keystone.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_keystone import KeystoneValidator from util.morecollections import AttrDict @@ -13,7 +14,7 @@ from test.fixtures import * ({'AUTHENTICATION_TYPE': 'Database'}), ]) def test_validate_noop(unvalidated_config, app): - KeystoneValidator.validate(unvalidated_config, None, None) + KeystoneValidator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('unvalidated_config', [ ({'AUTHENTICATION_TYPE': 'Keystone'}), @@ -25,7 +26,7 @@ def test_validate_noop(unvalidated_config, app): ]) def test_invalid_config(unvalidated_config, app): with pytest.raises(ConfigValidationException): - KeystoneValidator.validate(unvalidated_config, None, None) + KeystoneValidator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('username, password, expected_exception', [ @@ -45,8 +46,12 @@ def test_validated_keystone(username, password, expected_exception, app): config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass' config['KEYSTONE_ADMIN_TENANT'] = 'admintenant' + unvalidated_config = ValidatorContext(config) + unvalidated_config.user = AttrDict(dict(username=username)) + unvalidated_config.user_password = password + if expected_exception is not None: with pytest.raises(ConfigValidationException): - KeystoneValidator.validate(config, AttrDict(dict(username=username)), password) + KeystoneValidator.validate(unvalidated_config) else: - KeystoneValidator.validate(config, AttrDict(dict(username=username)), password) + KeystoneValidator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_ldap.py b/util/config/validators/test/test_validate_ldap.py index cdffce467..ee6f971d2 100644 --- a/util/config/validators/test/test_validate_ldap.py +++ b/util/config/validators/test/test_validate_ldap.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_ldap import LDAPValidator from util.morecollections import AttrDict @@ -7,13 +8,16 @@ from util.morecollections import AttrDict from test.test_ldap import mock_ldap from test.fixtures import * +from app import config_provider @pytest.mark.parametrize('unvalidated_config', [ ({}), ({'AUTHENTICATION_TYPE': 'Database'}), ]) def test_validate_noop(unvalidated_config, app): - LDAPValidator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.config_provider = config_provider + LDAPValidator.validate(config) @pytest.mark.parametrize('unvalidated_config', [ ({'AUTHENTICATION_TYPE': 'LDAP'}), @@ -21,7 +25,9 @@ def test_validate_noop(unvalidated_config, app): ]) def test_invalid_config(unvalidated_config, app): with pytest.raises(ConfigValidationException): - LDAPValidator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.config_provider = config_provider + LDAPValidator.validate(config) @pytest.mark.parametrize('uri', [ @@ -39,7 +45,9 @@ def test_invalid_uri(uri, app): config['LDAP_URI'] = uri with pytest.raises(ConfigValidationException): - LDAPValidator.validate(config, None, None) + config = ValidatorContext(config) + config.config_provider = config_provider + LDAPValidator.validate(config) @pytest.mark.parametrize('username, password, expected_exception', [ @@ -56,10 +64,15 @@ def test_validated_ldap(username, password, expected_exception, app): config['LDAP_ADMIN_PASSWD'] = 'password' config['LDAP_USER_RDN'] = ['ou=employees'] + unvalidated_config = ValidatorContext(config) + unvalidated_config.user = AttrDict(dict(username=username)) + unvalidated_config.user_password = password + unvalidated_config.config_provider = config_provider + if expected_exception is not None: with pytest.raises(ConfigValidationException): with mock_ldap(): - LDAPValidator.validate(config, AttrDict(dict(username=username)), password) + LDAPValidator.validate(unvalidated_config) else: with mock_ldap(): - LDAPValidator.validate(config, AttrDict(dict(username=username)), password) + LDAPValidator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_oidc.py b/util/config/validators/test/test_validate_oidc.py index 684ffef34..e884d4480 100644 --- a/util/config/validators/test/test_validate_oidc.py +++ b/util/config/validators/test/test_validate_oidc.py @@ -3,7 +3,9 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session from oauth.oidc import OIDC_WELLKNOWN +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_oidc import OIDCLoginValidator @@ -19,7 +21,7 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, app): validator = OIDCLoginValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_oidc_login(app): url_hit = [False] @@ -33,13 +35,16 @@ def test_validate_oidc_login(app): with HTTMock(handler): validator = OIDCLoginValidator() - validator.validate({ + unvalidated_config = ValidatorContext({ 'SOMETHING_LOGIN_CONFIG': { 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', 'OIDC_SERVER': 'http://someserver', 'DEBUGGING': True, # Allows for HTTP. }, - }, None, None) + }) + unvalidated_config.http_client = build_requests_session() + + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_redis.py b/util/config/validators/test/test_validate_redis.py index c6d1c2498..39a1ce5ee 100644 --- a/util/config/validators/test/test_validate_redis.py +++ b/util/config/validators/test/test_validate_redis.py @@ -5,10 +5,13 @@ from mock import patch from mockredis import mock_strict_redis_client +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_redis import RedisValidator from test.fixtures import * +from util.morecollections import AttrDict + @pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [ ({}, None, None, False, ConfigValidationException), @@ -19,8 +22,13 @@ from test.fixtures import * def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app): with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client): validator = RedisValidator() + unvalidated_config = ValidatorContext(unvalidated_config) + + unvalidated_config.user = AttrDict(dict(username=user)) + unvalidated_config.user_password = user_password + if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) else: - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_secscan.py b/util/config/validators/test/test_validate_secscan.py index 6232f3156..550fb4cfc 100644 --- a/util/config/validators/test/test_validate_secscan.py +++ b/util/config/validators/test/test_validate_secscan.py @@ -1,6 +1,8 @@ import pytest -from util.config.validators import ConfigValidationException +from config import build_requests_session +from util.config import URLSchemeAndHostname +from util.config.validator import ValidatorContext from util.config.validators.validate_secscan import SecurityScannerValidator from util.secscan.fake import fake_security_scanner @@ -10,7 +12,11 @@ from test.fixtures import * ({'DISTRIBUTED_STORAGE_PREFERENCE': []}), ]) def test_validate_noop(unvalidated_config, app): - SecurityScannerValidator.validate(unvalidated_config, None, None) + unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True) + unvalidated_config.http_client = build_requests_session() + unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000') + + SecurityScannerValidator.validate(unvalidated_config) @pytest.mark.parametrize('unvalidated_config, expected_error', [ @@ -29,9 +35,13 @@ def test_validate_noop(unvalidated_config, app): }, None), ]) def test_validate(unvalidated_config, expected_error, app): + unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True) + unvalidated_config.http_client = build_requests_session() + unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000') + with fake_security_scanner(hostname='fakesecurityscanner'): if expected_error is not None: with pytest.raises(expected_error): - SecurityScannerValidator.validate(unvalidated_config, None, None) + SecurityScannerValidator.validate(unvalidated_config) else: - SecurityScannerValidator.validate(unvalidated_config, None, None) + SecurityScannerValidator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_signer.py b/util/config/validators/test/test_validate_signer.py index 4ee01cd9f..f45f91c11 100644 --- a/util/config/validators/test/test_validate_signer.py +++ b/util/config/validators/test/test_validate_signer.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_signer import SignerValidator @@ -14,6 +15,6 @@ def test_validate_signer(unvalidated_config, expected, app): validator = SignerValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) else: - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) diff --git a/util/config/validators/test/test_validate_ssl.py b/util/config/validators/test/test_validate_ssl.py index c7ec334be..8ea7f3297 100644 --- a/util/config/validators/test/test_validate_ssl.py +++ b/util/config/validators/test/test_validate_ssl.py @@ -3,11 +3,13 @@ import pytest from mock import patch from tempfile import NamedTemporaryFile +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES from test.test_ssl_util import generate_test_cert from test.fixtures import * +from app import config_provider @pytest.mark.parametrize('unvalidated_config', [ ({}), @@ -16,7 +18,7 @@ from test.fixtures import * ]) def test_skip_validate_ssl(unvalidated_config, app): validator = SSLValidator() - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('cert, expected_error, error_message', [ @@ -54,11 +56,13 @@ def test_validate_ssl(cert, expected_error, error_message, app): with patch('app.config_provider.volume_file_exists', return_true): with patch('app.config_provider.get_volume_file', get_volume_file): validator = SSLValidator() + config = ValidatorContext(config) + config.config_provider = config_provider if expected_error is not None: with pytest.raises(expected_error) as ipe: - validator.validate(config, None, None) + validator.validate(config) assert ipe.value.message == error_message else: - validator.validate(config, None, None) + validator.validate(config) diff --git a/util/config/validators/test/test_validate_storage.py b/util/config/validators/test/test_validate_storage.py index f360eab3c..007452c7f 100644 --- a/util/config/validators/test/test_validate_storage.py +++ b/util/config/validators/test/test_validate_storage.py @@ -1,6 +1,7 @@ import moto import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_storage import StorageValidator @@ -16,15 +17,15 @@ def test_validate_storage(unvalidated_config, expected, app): validator = StorageValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) else: - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_s3_storage(app): validator = StorageValidator() with moto.mock_s3(): with pytest.raises(ConfigValidationException) as ipe: - validator.validate({ + validator.validate(ValidatorContext({ 'DISTRIBUTED_STORAGE_CONFIG': { 'default': ('S3Storage', { 's3_access_key': 'invalid', @@ -33,6 +34,6 @@ def test_validate_s3_storage(app): 'storage_path': '' }), } - }, None, None) + })) assert ipe.value.message == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found' \ No newline at end of file diff --git a/util/config/validators/test/test_validate_timemachine.py b/util/config/validators/test/test_validate_timemachine.py index e1da63a45..1c3b29ba5 100644 --- a/util/config/validators/test/test_validate_timemachine.py +++ b/util/config/validators/test/test_validate_timemachine.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_timemachine import TimeMachineValidator @@ -7,7 +8,7 @@ from util.config.validators.validate_timemachine import TimeMachineValidator ({}), ]) def test_validate_noop(unvalidated_config): - TimeMachineValidator.validate(unvalidated_config, None, None) + TimeMachineValidator.validate(ValidatorContext(unvalidated_config)) from test.fixtures import * @@ -25,7 +26,7 @@ def test_validate(default_exp, options, expected_exception, app): if expected_exception is not None: with pytest.raises(ConfigValidationException) as cve: - TimeMachineValidator.validate(config, None, None) + TimeMachineValidator.validate(ValidatorContext(config)) assert str(cve.value) == str(expected_exception) else: - TimeMachineValidator.validate(config, None, None) + TimeMachineValidator.validate(ValidatorContext(config)) diff --git a/util/config/validators/test/test_validate_torrent.py b/util/config/validators/test/test_validate_torrent.py index badd08198..1ad3664b0 100644 --- a/util/config/validators/test/test_validate_torrent.py +++ b/util/config/validators/test/test_validate_torrent.py @@ -2,6 +2,8 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_torrent import BittorrentValidator @@ -23,8 +25,14 @@ def test_validate_torrent(unvalidated_config, expected, app): validator = BittorrentValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.http_client = build_requests_session() + + validator.validate(config) assert not announcer_hit[0] else: - validator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.http_client = build_requests_session() + + validator.validate(config) assert announcer_hit[0] diff --git a/util/config/validators/validate_bitbucket_trigger.py b/util/config/validators/validate_bitbucket_trigger.py index 04ae1de04..a0becd50d 100644 --- a/util/config/validators/validate_bitbucket_trigger.py +++ b/util/config/validators/validate_bitbucket_trigger.py @@ -23,7 +23,7 @@ class BitbucketTriggerValidator(BaseValidator): key = trigger_config['CONSUMER_KEY'] secret = trigger_config['CONSUMER_SECRET'] - callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url(validator_context.scheme_and_hostname)) + callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url_from_scheme_hostname(validator_context.url_scheme_and_hostname)) bitbucket_client = BitBucket(key, secret, callback_url) (result, _, _) = bitbucket_client.get_authorization_url() diff --git a/util/config/validators/validate_google_login.py b/util/config/validators/validate_google_login.py index 7af89ff0c..6481e4225 100644 --- a/util/config/validators/validate_google_login.py +++ b/util/config/validators/validate_google_login.py @@ -21,7 +21,6 @@ class GoogleLoginValidator(BaseValidator): raise ConfigValidationException('Missing Client Secret') oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG') - # TODO(sam): the google oauth doesn't need the app config, but when refactoring pass in the URLSchemeandHostname result = oauth.validate_client_id_and_secret(client) if not result: raise ConfigValidationException('Invalid client id or client secret') diff --git a/util/config/validators/validate_jwt.py b/util/config/validators/validate_jwt.py index 83b3bfe19..942956174 100644 --- a/util/config/validators/validate_jwt.py +++ b/util/config/validators/validate_jwt.py @@ -1,4 +1,4 @@ -from _init import OVERRIDE_CONFIG_DIRECTORY +import os from data.users.externaljwt import ExternalJWTAuthN from util.config.validators import BaseValidator, ConfigValidationException @@ -13,6 +13,7 @@ class JWTAuthValidator(BaseValidator): user_password = validator_context.user_password http_client = validator_context.http_client jwt_auth_max = validator_context.jwt_auth_max + config_provider = validator_context.config_provider if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT': return @@ -29,10 +30,13 @@ class JWTAuthValidator(BaseValidator): if not issuer: raise ConfigValidationException('Missing JWT Issuer ID') + + override_config_directory = os.path.join(config_provider.get_config_root(), 'stack/') + # Try to instatiate the JWT authentication mechanism. This will raise an exception if # the key cannot be found. users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer, - OVERRIDE_CONFIG_DIRECTORY, + override_config_directory, http_client, jwt_auth_max, public_key_path=public_key_path, diff --git a/util/config/validators/validate_ldap.py b/util/config/validators/validate_ldap.py index febb5aec1..331cd87d3 100644 --- a/util/config/validators/validate_ldap.py +++ b/util/config/validators/validate_ldap.py @@ -5,7 +5,6 @@ import subprocess from data.users import LDAP_CERT_FILENAME from data.users.externalldap import LDAPConnection, LDAPUsers from util.config.validators import BaseValidator, ConfigValidationException -from _init import CONF_DIR, config_provider class LDAPValidator(BaseValidator): name = "ldap" @@ -16,12 +15,14 @@ class LDAPValidator(BaseValidator): config = validator_context.config user = validator_context.user user_password = validator_context.user_password + config_provider = validator_context.config_provider + if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP': return # If there is a custom LDAP certificate, then reinstall the certificates for the container. if config_provider.volume_file_exists(LDAP_CERT_FILENAME): - subprocess.check_call([os.path.join(CONF_DIR, 'init/certs_install.sh')]) + subprocess.check_call([os.path.join(config_provider.get_config_root(), '../init/certs_install.sh')]) # Note: raises ldap.INVALID_CREDENTIALS on failure admin_dn = config.get('LDAP_ADMIN_DN') diff --git a/util/config/validators/validate_secscan.py b/util/config/validators/validate_secscan.py index 0e1eac870..9f7c2d67f 100644 --- a/util/config/validators/validate_secscan.py +++ b/util/config/validators/validate_secscan.py @@ -12,14 +12,18 @@ class SecurityScannerValidator(BaseValidator): """ Validates the configuration for talking to a Quay Security Scanner. """ config = validator_context.config client = validator_context.http_client - app = None #TODO(sam) validate with joey's pr about security scanner api + feature_sec_scanner = validator_context.feature_sec_scanner + is_testing = validator_context.is_testing - if not config.get('FEATURE_SECURITY_SCANNER', False): + server_hostname = validator_context.url_scheme_and_hostname.hostname + uri_creator = validator_context.uri_creator + + if not feature_sec_scanner: return - api = SecurityScannerAPI(app.config, config, None, client=client, skip_validation=True) + api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator) - if not config.get('TESTING', False): + if not is_testing: # Generate a temporary Quay key to use for signing the outgoing requests. setup_jwt_proxy() diff --git a/util/config/validators/validate_signer.py b/util/config/validators/validate_signer.py index 241ecf227..f151ab19d 100644 --- a/util/config/validators/validate_signer.py +++ b/util/config/validators/validate_signer.py @@ -1,6 +1,5 @@ from StringIO import StringIO -from _init import config_provider from util.config.validators import BaseValidator, ConfigValidationException from util.security.signing import SIGNING_ENGINES @@ -11,6 +10,7 @@ class SignerValidator(BaseValidator): def validate(cls, validator_context): """ Validates the GPG public+private key pair used for signing converted ACIs. """ config = validator_context.config + config_provider = validator_context.config_provider if config.get('SIGNING_ENGINE') is None: return diff --git a/util/config/validators/validate_ssl.py b/util/config/validators/validate_ssl.py index b76645e82..326754184 100644 --- a/util/config/validators/validate_ssl.py +++ b/util/config/validators/validate_ssl.py @@ -1,4 +1,3 @@ -from _init import config_provider from util.config.validators import BaseValidator, ConfigValidationException from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException @@ -11,6 +10,7 @@ class SSLValidator(BaseValidator): def validate(cls, validator_context): """ Validates the SSL configuration (if enabled). """ config = validator_context.config + config_provider = validator_context.config_provider # Skip if non-SSL. if config.get('PREFERRED_URL_SCHEME', 'http') != 'https': diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index bb3c0e685..e885e5f7c 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -1,4 +1,3 @@ -from _init import config_provider from storage import get_storage_driver from util.config.validators import BaseValidator, ConfigValidationException @@ -12,10 +11,12 @@ class StorageValidator(BaseValidator): config = validator_context.config client = validator_context.http_client ip_resolver = validator_context.ip_resolver + config_provider = validator_context.config_provider + # replication_enabled = app.config.get('FEATURE_STORAGE_REPLICATION', False) replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False) - providers = _get_storage_providers(config, ip_resolver).items() + providers = _get_storage_providers(config, ip_resolver, config_provider).items() if not providers: raise ConfigValidationException('Storage configuration required') @@ -35,7 +36,7 @@ class StorageValidator(BaseValidator): raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg)) -def _get_storage_providers(config, ip_resolver): +def _get_storage_providers(config, ip_resolver, config_provider): storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {}) drivers = {} diff --git a/util/secscan/__init__.py b/util/secscan/__init__.py index 7871f28b4..1e3ac04aa 100644 --- a/util/secscan/__init__.py +++ b/util/secscan/__init__.py @@ -107,12 +107,3 @@ def get_priority_for_index(index): return priority return 'Unknown' - - -def create_url_from_app(app): - """ - Higher order function that returns a function that when called, will generate a url for that given app - :param app: Flask app - :return: - type: Flask -> (str -> url) - """ diff --git a/util/secscan/api.py b/util/secscan/api.py index 4bae2034d..25ef36807 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -7,16 +7,12 @@ from urlparse import urljoin import requests -from flask import url_for - from data.database import CloseForLongOperation from data import model from data.model.storage import get_storage_locations -from util import get_app_url, slash_join from util.abchelpers import nooper from util.failover import failover, FailoverException from util.secscan.validator import SecurityConfigValidator -from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config from util.security.registry_jwt import generate_bearer_token, build_context_and_subject from _init import CONF_DIR @@ -70,16 +66,16 @@ def compute_layer_id(layer): class SecurityScannerAPI(object): """ Helper class for talking to the Security Scan service (usually Clair). """ - def __init__(self, app, config, storage, client=None, skip_validation=False): + def __init__(self, config, storage, server_hostname=None, client=None, skip_validation=False, uri_creator=None, instance_keys=None): feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False) has_valid_config = skip_validation if not skip_validation and feature_enabled: - config_validator = SecurityConfigValidator(config) + config_validator = SecurityConfigValidator(feature_enabled, config.get('SECURITY_SCANNER_ENDPOINT')) has_valid_config = config_validator.valid() if feature_enabled and has_valid_config: - self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client) + self.state = ImplementedSecurityScannerAPI(config, storage, server_hostname, client=client, uri_creator=uri_creator, instance_keys=instance_keys) else: self.state = NoopSecurityScannerAPI() @@ -150,20 +146,25 @@ class NoopSecurityScannerAPI(SecurityScannerAPIInterface): class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): """ Helper class for talking to the Security Scan service (Clair). """ - def __init__(self, app_config, config, storage, client=None): - self._app_config = app_config + # TODO(sam) refactor this to not take an app config, and instead just the things it needs as a config object + def __init__(self, config, storage, server_hostname, client=None, uri_creator=None, instance_keys=None): self._config = config - self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app_config)) - self._client = client or config['HTTPCLIENT'] + self._instance_keys = instance_keys + self._client = client self._storage = storage + self._server_hostname = server_hostname self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE'] self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2) + self._uri_creator = uri_creator def _get_image_url_and_auth(self, image): """ Returns a tuple of the url and the auth header value that must be used to fetch the layer data itself. If the image can't be addressed, we return None. """ + if self._instance_keys is None: + raise Exception('No Instance keys provided to Security Scanner API') + path = model.storage.get_layer_path(image.storage) locations = self._default_storage_locations @@ -183,7 +184,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): repository_and_namespace = '/'.join([namespace_name, repo_name]) # Generate the JWT which will authorize this - audience = self._app_config['SERVER_HOSTNAME'] + audience = self._server_hostname context, subject = build_context_and_subject() access = [{ 'type': 'repository', @@ -195,10 +196,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): TOKEN_VALIDITY_LIFETIME_S, self._instance_keys) auth_header = 'Bearer ' + auth_token - with self._app.test_request_context('/'): - relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace, - digest=image.storage.content_checksum) - uri = urljoin(get_app_url(self._config), relative_layer_url) + uri = self._uri_creator(repository_and_namespace, image.storage.content_checksum) return uri, auth_header diff --git a/util/secscan/validator.py b/util/secscan/validator.py index d8eb61c01..1eaa1244b 100644 --- a/util/secscan/validator.py +++ b/util/secscan/validator.py @@ -1,28 +1,27 @@ import logging -import features - logger = logging.getLogger(__name__) class SecurityConfigValidator(object): """ Helper class for validating the security scanner configuration. """ - def __init__(self, config): - if not features.SECURITY_SCANNER: + def __init__(self, feature_sec_scan, sec_scan_endpoint): + if not feature_sec_scan: return - self._config = config + self._feature_sec_scan = feature_sec_scan + self._sec_scan_endpoint = sec_scan_endpoint def valid(self): - if not features.SECURITY_SCANNER: + if not self._feature_sec_scan: return False - if self._config.get('SECURITY_SCANNER_ENDPOINT') is None: + if self._sec_scan_endpoint is None: logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration') return False - endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT') + endpoint = self._sec_scan_endpoint if not endpoint.startswith('http://') and not endpoint.startswith('https://'): logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https') return False diff --git a/util/security/instancekeys.py b/util/security/instancekeys.py index c5b12c3b0..75269552c 100644 --- a/util/security/instancekeys.py +++ b/util/security/instancekeys.py @@ -1,4 +1,3 @@ -from collections import namedtuple from cachetools import lru_cache from data import model from util.expiresdict import ExpiresDict, ExpiresEntry @@ -26,10 +25,9 @@ class InstanceKeys(object): """ InstanceKeys defines a helper class for interacting with the Quay instance service keys used for JWT signing of registry tokens as well as requests from Quay to other services such as Clair. Each container will have a single registered instance key. - :param keys_context: InstanceKeysContext """ - def __init__(self, keys_context): - self.keys_context = keys_context + def __init__(self, app): + self.app = app self.instance_keys = ExpiresDict(self._load_instance_keys) def clear_cache(self): @@ -47,24 +45,24 @@ class InstanceKeys(object): @property def service_name(self): """ Returns the name of the instance key's service (i.e. 'quay'). """ - return self.keys_context.instance_key_service + return self.app.config['INSTANCE_SERVICE_KEY_SERVICE'] @property def service_key_expiration(self): """ Returns the defined expiration for instance service keys, in minutes. """ - return self.keys_context.service_key_expiration + return self.app.config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120) @property @lru_cache(maxsize=1) def local_key_id(self): """ Returns the ID of the local instance service key. """ - return _load_file_contents(self.keys_context.service_key_kid_location) + return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_KID_LOCATION']) @property @lru_cache(maxsize=1) def local_private_key(self): """ Returns the private key of the local instance service key. """ - return _load_file_contents(self.keys_context.service_key_location) + return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_LOCATION']) def get_service_key_public_key(self, kid): """ Returns the public key associated with the given instance service key or None if none. """ @@ -79,15 +77,3 @@ def _load_file_contents(path): """ Returns the contents of the specified file path. """ with open(path) as f: return f.read() - - -InstanceKeysContext = namedtuple('InstanceKeysContext', ['instance_key_service', - 'service_key_expiration', - 'service_key_kid_location', - 'service_key_location']) - -def instance_keys_context_from_app_config(app_config): - return InstanceKeysContext(app_config['INSTANCE_SERVICE_KEY_SERVICE'], - app_config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120), - app_config['INSTANCE_SERVICE_KEY_KID_LOCATION'], - app_config['INSTANCE_SERVICE_KEY_LOCATION']) diff --git a/util/tufmetadata/api.py b/util/tufmetadata/api.py index 0f9f123b7..9c039477a 100644 --- a/util/tufmetadata/api.py +++ b/util/tufmetadata/api.py @@ -11,7 +11,7 @@ import requests from data.database import CloseForLongOperation from util.abchelpers import nooper from util.failover import failover, FailoverException -from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config +from util.security.instancekeys import InstanceKeys from util.security.registry_jwt import (build_context_and_subject, generate_bearer_token, SIGNER_TUF_ROOT) @@ -108,7 +108,7 @@ class NoopTUFMetadataAPI(TUFMetadataAPIInterface): class ImplementedTUFMetadataAPI(TUFMetadataAPIInterface): def __init__(self, app, config, client=None): self._app = app - self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config)) + self._instance_keys = InstanceKeys(app) self._config = config self._client = client or config['HTTPCLIENT'] self._gun_prefix = config['TUF_GUN_PREFIX'] or config['SERVER_HOSTNAME'] diff --git a/workers/securityworker/securityworker.py b/workers/securityworker/securityworker.py index 552d7fd2f..100308acf 100644 --- a/workers/securityworker/securityworker.py +++ b/workers/securityworker/securityworker.py @@ -19,7 +19,7 @@ DEFAULT_INDEXING_INTERVAL = 30 class SecurityWorker(Worker): def __init__(self): super(SecurityWorker, self).__init__() - validator = SecurityConfigValidator(app.config) + validator = SecurityConfigValidator(app.config.get('FEATURE_SECURITY_SCANNER', False), app.config.get('SECURITY_SCANNER_ENDPOINT')) if not validator.valid(): logger.warning('Failed to validate security scan configuration') return