diff --git a/_init.py b/_init.py index 804323555..3acbc3742 100644 --- a/_init.py +++ b/_init.py @@ -2,6 +2,8 @@ import os import re import subprocess +from util.config.provider import get_config_provider + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) CONF_DIR = os.getenv("QUAYCONF", os.path.join(ROOT_DIR, "conf/")) @@ -10,6 +12,14 @@ STATIC_LDN_DIR = os.path.join(STATIC_DIR, 'ldn/') STATIC_FONTS_DIR = os.path.join(STATIC_DIR, 'fonts/') TEMPLATE_DIR = os.path.join(ROOT_DIR, 'templates/') +IS_TESTING = 'TEST' in os.environ +IS_KUBERNETES = 'KUBERNETES_SERVICE_HOST' in os.environ +OVERRIDE_CONFIG_DIRECTORY = os.path.join(CONF_DIR, 'stack/') + + +config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py', + testing=IS_TESTING, kubernetes=IS_KUBERNETES) + def _get_version_number_changelog(): try: diff --git a/app.py b/app.py index 90f5771b6..7845b43fd 100644 --- a/app.py +++ b/app.py @@ -13,7 +13,9 @@ from flask_principal import Principal from jwkest.jwk import RSAKey import features -from _init import CONF_DIR + +from _init import config_provider, CONF_DIR, IS_KUBERNETES, IS_TESTING, OVERRIDE_CONFIG_DIRECTORY + from auth.auth_context import get_authenticated_user from avatars.avatars import Avatar from buildman.manager.buildcanceller import BuildCanceller @@ -33,15 +35,16 @@ from oauth.services.github import GithubOAuthService from oauth.services.gitlab import GitLabOAuthService from oauth.loginmanager import OAuthLoginManager from storage import Storage +from util.config import URLSchemeAndHostname from util.log import filter_logs from util import get_app_url +from util.secscan.secscan_util import get_blob_download_uri_getter from util.ipresolver import IPResolver from util.saas.analytics import Analytics from util.saas.useranalytics import UserAnalytics from util.saas.exceptionlog import Sentry from util.names import urn_generator from util.config.configutil import generate_secret_key -from util.config.provider import get_config_provider from util.config.superusermanager import SuperUserManager from util.label_validator import LabelValidator from util.metrics.metricqueue import MetricQueue @@ -53,7 +56,6 @@ from util.security.instancekeys import InstanceKeys from util.security.signing import Signer -OVERRIDE_CONFIG_DIRECTORY = os.path.join(CONF_DIR, 'stack/') OVERRIDE_CONFIG_YAML_FILENAME = os.path.join(CONF_DIR, 'stack/config.yaml') OVERRIDE_CONFIG_PY_FILENAME = os.path.join(CONF_DIR, 'stack/config.py') @@ -65,10 +67,8 @@ app = Flask(__name__) logger = logging.getLogger(__name__) # Instantiate the configuration. -is_testing = 'TEST' in os.environ -is_kubernetes = 'KUBERNETES_SERVICE_HOST' in os.environ -config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py', - testing=is_testing, kubernetes=is_kubernetes) +is_testing = IS_TESTING +is_kubernetes = IS_KUBERNETES if is_testing: from test.testconfig import TestConfig @@ -230,7 +230,11 @@ namespace_gc_queue = WorkQueue(app.config['NAMESPACE_GC_QUEUE_NAME'], tf, has_na all_queues = [image_replication_queue, dockerfile_build_queue, notification_queue, secscan_notification_queue, chunk_cleanup_queue, namespace_gc_queue] -secscan_api = SecurityScannerAPI(app, app.config, storage) +url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) +secscan_api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'], + uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname), + instance_keys=instance_keys) + tuf_metadata_api = TUFMetadataAPI(app, app.config) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 836f09f80..11984d5ab 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -7,7 +7,7 @@ import subprocess from flask import abort -from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY +from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver from auth.permissions import SuperUserPermission from auth.auth_context import get_authenticated_user from data.database import configure @@ -20,7 +20,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_ from endpoints.common import common_login from util.config.configutil import add_enterprise_config_defaults from util.config.database import sync_database_with_config -from util.config.validator import validate_service_for_config, is_valid_config_upload_filename +from util.config.validator import validate_service_for_config, is_valid_config_upload_filename, ValidatorContext import features @@ -405,6 +405,10 @@ class SuperUserConfigValidate(ApiResource): # this is also safe since this method does not access any information not given in the request. if not config_provider.config_exists() or SuperUserPermission().can(): config = request.get_json()['config'] - return validate_service_for_config(service, config, request.get_json().get('password', '')) + validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''), + ip_resolver=ip_resolver, + config_provider=config_provider) + + return validate_service_for_config(service, validator_context) abort(403) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index fc68019ea..dde7bb15f 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -12,7 +12,7 @@ from peewee import IntegrityError import features from app import (app, billing as stripe, authentication, avatar, user_analytics, all_queues, - oauth_login, namespace_gc_queue, ip_resolver) + oauth_login, namespace_gc_queue, ip_resolver, url_scheme_and_hostname) from auth import scopes from auth.auth_context import get_authenticated_user @@ -785,7 +785,7 @@ class ExternalLoginInformation(ApiResource): try: login_scopes = login_service.get_login_scopes() - auth_url = login_service.get_auth_url(app.config, redirect_suffix, csrf_token, login_scopes) + auth_url = login_service.get_auth_url(url_scheme_and_hostname, redirect_suffix, csrf_token, login_scopes) return {'auth_url': auth_url} except DiscoveryFailureException as dfe: logger.exception('Could not discovery OAuth endpoint information') diff --git a/endpoints/oauth/login.py b/endpoints/oauth/login.py index 642cd068b..2a0ade3af 100644 --- a/endpoints/oauth/login.py +++ b/endpoints/oauth/login.py @@ -8,7 +8,7 @@ from peewee import IntegrityError import features -from app import app, analytics, get_app_url, oauth_login, authentication +from app import app, analytics, get_app_url, oauth_login, authentication, url_scheme_and_hostname from auth.auth_context import get_authenticated_user from auth.decorators import require_session_login from data import model @@ -251,7 +251,7 @@ def _register_service(login_service): # Redirect to the normal OAuth flow again, so that the user can now create an account. csrf_token = generate_csrf_token(OAUTH_CSRF_TOKEN_NAME) login_scopes = login_service.get_login_scopes() - auth_url = login_service.get_auth_url(app.config, '', csrf_token, login_scopes) + auth_url = login_service.get_auth_url(url_scheme_and_hostname, '', csrf_token, login_scopes) return redirect(auth_url) @require_session_login diff --git a/oauth/base.py b/oauth/base.py index 12ca4dfef..2ed0af706 100644 --- a/oauth/base.py +++ b/oauth/base.py @@ -6,7 +6,7 @@ import urlparse from abc import ABCMeta, abstractmethod from six import add_metaclass -from util import get_app_url +from util.config import URLSchemeAndHostname logger = logging.getLogger(__name__) @@ -73,7 +73,7 @@ class OAuthService(object): pass @abstractmethod - def validate_client_id_and_secret(self, http_client, app_config): + def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname): """ Performs validation of the client ID and secret, raising an exception on failure. """ pass @@ -98,9 +98,10 @@ class OAuthService(object): """ return self.config.get('LOGIN_BINDING_FIELD', None) - def get_auth_url(self, app_config, redirect_suffix, csrf_token, scopes): + def get_auth_url(self, url_scheme_and_hostname, redirect_suffix, csrf_token, scopes): """ Retrieves the authorization URL for this login service. """ - redirect_uri = '%s/oauth2/%s/callback%s' % (get_app_url(app_config), self.service_id(), + redirect_uri = '%s/oauth2/%s/callback%s' % (url_scheme_and_hostname.get_url(), + self.service_id(), redirect_suffix) params = { 'client_id': self.client_id(), @@ -111,9 +112,9 @@ class OAuthService(object): return self.authorize_endpoint().with_params(params).to_url() - def get_redirect_uri(self, app_config, redirect_suffix=''): - return '%s://%s/oauth2/%s/callback%s' % (app_config['PREFERRED_URL_SCHEME'], - app_config['SERVER_HOSTNAME'], + def get_redirect_uri(self, url_scheme_and_hostname, redirect_suffix=''): + return '%s://%s/oauth2/%s/callback%s' % (url_scheme_and_hostname.url_scheme, + url_scheme_and_hostname.hostname, self.service_id(), redirect_suffix) @@ -153,10 +154,11 @@ class OAuthService(object): def exchange_code(self, app_config, http_client, code, form_encode=False, redirect_suffix='', client_auth=False): """ Exchanges an OAuth access code for associated OAuth token and other data. """ + url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app_config) payload = { 'code': code, 'grant_type': 'authorization_code', - 'redirect_uri': self.get_redirect_uri(app_config, redirect_suffix) + 'redirect_uri': self.get_redirect_uri(url_scheme_and_hostname, redirect_suffix) } headers = { diff --git a/oauth/oidc.py b/oauth/oidc.py index e6292530f..892929b29 100644 --- a/oauth/oidc.py +++ b/oauth/oidc.py @@ -97,9 +97,9 @@ class OIDCLoginService(OAuthService): def validate(self): return bool(self.get_login_scopes()) - def validate_client_id_and_secret(self, http_client, app_config): + def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname): # TODO: find a way to verify client secret too. - check_auth_url = http_client.get(self.get_auth_url(app_config, '', '', [])) + check_auth_url = http_client.get(self.get_auth_url(url_scheme_and_hostname, '', '', [])) if check_auth_url.status_code // 100 != 2: raise Exception('Got non-200 status code for authorization endpoint') diff --git a/oauth/services/github.py b/oauth/services/github.py index 6bc9350ff..b7eaca04c 100644 --- a/oauth/services/github.py +++ b/oauth/services/github.py @@ -75,7 +75,7 @@ class GithubOAuthService(OAuthLoginService): def orgs_endpoint(self): return slash_join(self._api_endpoint(), 'user/orgs') - def validate_client_id_and_secret(self, http_client, app_config): + def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname): # First: Verify that the github endpoint is actually Github by checking for the # X-GitHub-Request-Id here. api_endpoint = self._api_endpoint() diff --git a/oauth/services/gitlab.py b/oauth/services/gitlab.py index 1ee2f90ed..9b0dcc2ec 100644 --- a/oauth/services/gitlab.py +++ b/oauth/services/gitlab.py @@ -29,13 +29,13 @@ class GitLabOAuthService(OAuthService): def token_endpoint(self): return OAuthEndpoint(slash_join(self._endpoint(), '/oauth/token')) - def validate_client_id_and_secret(self, http_client, app_config): + def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname): # We validate the client ID and secret by hitting the OAuth token exchange endpoint with # the real client ID and secret, but a fake auth code to exchange. Gitlab's implementation will # return `invalid_client` as the `error` if the client ID or secret is invalid; otherwise, it # will return another error. url = self.token_endpoint().to_url() - redirect_uri = self.get_redirect_uri(app_config, redirect_suffix='trigger') + redirect_uri = self.get_redirect_uri(url_scheme_and_hostname, redirect_suffix='trigger') data = { 'code': 'fakecode', 'client_id': self.client_id(), diff --git a/oauth/services/google.py b/oauth/services/google.py index a22964bb6..515a5b5dd 100644 --- a/oauth/services/google.py +++ b/oauth/services/google.py @@ -41,7 +41,7 @@ class GoogleOAuthService(OAuthLoginService): def requires_form_encoding(self): return True - def validate_client_id_and_secret(self, http_client, app_config): + def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname): # To verify the Google client ID and secret, we hit the # https://www.googleapis.com/oauth2/v3/token endpoint with an invalid request. If the client # ID or secret are invalid, we get returned a 403 Unauthorized. Otherwise, we get returned diff --git a/oauth/test/test_oidc.py b/oauth/test/test_oidc.py index bd57defe2..1309060e3 100644 --- a/oauth/test/test_oidc.py +++ b/oauth/test/test_oidc.py @@ -13,6 +13,8 @@ from Crypto.PublicKey import RSA from jwkest.jwk import RSAKey from oauth.oidc import OIDCLoginService, OAuthLoginException +from util.config import URLSchemeAndHostname + @pytest.fixture(scope='module') # Slow to generate, only do it once. def signing_key(): @@ -277,7 +279,8 @@ def test_auth_url(oidc_service, discovery_handler, http_client, authorize_handle config = {'PREFERRED_URL_SCHEME': 'https', 'SERVER_HOSTNAME': 'someserver'} with HTTMock(discovery_handler, authorize_handler): - auth_url = oidc_service.get_auth_url(config, '', 'some csrf token', ['one', 'two']) + url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(config) + auth_url = oidc_service.get_auth_url(url_scheme_and_hostname, '', 'some csrf token', ['one', 'two']) # Hit the URL and ensure it works. result = http_client.get(auth_url).json() diff --git a/test/test_secscan.py b/test/test_secscan.py index 8ba559d95..e0a6d6534 100644 --- a/test/test_secscan.py +++ b/test/test_secscan.py @@ -2,17 +2,19 @@ import json import time import unittest -from app import app, storage, notification_queue +from app import app, storage, notification_queue, url_scheme_and_hostname from data import model from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION from endpoints.v2 import v2_bp from initdb import setup_database_for_testing, finished_database_for_testing from notifications.notificationevent import VulnerabilityFoundEvent +from util.secscan.secscan_util import get_blob_download_uri_getter from util.morecollections import AttrDict from util.secscan.api import SecurityScannerAPI, APIRequestFailure from util.secscan.analyzer import LayerAnalyzer from util.secscan.fake import fake_security_scanner from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult +from util.security.instancekeys import InstanceKeys from workers.security_notification_worker import SecurityNotificationWorker @@ -42,7 +44,12 @@ class TestSecurityScanner(unittest.TestCase): self.ctx = app.test_request_context() self.ctx.__enter__() - self.api = SecurityScannerAPI(app, app.config, storage) + + instance_keys = InstanceKeys(app) + self.api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'], + uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), + url_scheme_and_hostname), + instance_keys=instance_keys) def tearDown(self): storage.remove(['local_us'], 'supports_direct_download') diff --git a/util/__init__.py b/util/__init__.py index f4744897a..77f20ba27 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -1,8 +1,8 @@ + def get_app_url(config): """ Returns the application's URL, based on the given config. """ return '%s://%s' % (config['PREFERRED_URL_SCHEME'], config['SERVER_HOSTNAME']) - def slash_join(*args): """ Joins together strings and guarantees there is only one '/' in between the @@ -16,3 +16,4 @@ def slash_join(*args): args = [rmslash(path) for path in args] return '/'.join(args) + diff --git a/util/config/__init__.py b/util/config/__init__.py index e69de29bb..cbb177ff9 100644 --- a/util/config/__init__.py +++ b/util/config/__init__.py @@ -0,0 +1,29 @@ +class URLSchemeAndHostname: + """ + Immutable configuration for a given preferred url scheme (e.g. http or https), and a hostname (e.g. localhost:5000) + """ + def __init__(self, url_scheme, hostname): + self._url_scheme = url_scheme + self._hostname = hostname + + @classmethod + def from_app_config(cls, app_config): + """ + Helper method to instantiate class from app config, a frequent pattern + :param app_config: + :return: + """ + return cls(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME']) + + @property + def url_scheme(self): + return self._url_scheme + + @property + def hostname(self): + return self._hostname + + def get_url(self): + """ Returns the application's URL, based on the given url scheme and hostname. """ + return '%s://%s' % (self._url_scheme, self._hostname) + diff --git a/util/config/provider/basefileprovider.py b/util/config/provider/basefileprovider.py index 7d0572351..14fcdebb0 100644 --- a/util/config/provider/basefileprovider.py +++ b/util/config/provider/basefileprovider.py @@ -68,3 +68,7 @@ class BaseFileProvider(BaseProvider): return True return False + + def get_config_root(self): + return self.config_volume + diff --git a/util/config/provider/baseprovider.py b/util/config/provider/baseprovider.py index 052b32e76..02d693cf0 100644 --- a/util/config/provider/baseprovider.py +++ b/util/config/provider/baseprovider.py @@ -123,3 +123,8 @@ class BaseProvider(object): def get_volume_path(self, directory, filename): """ Helper for constructing relative file paths, which may differ between providers. For example, kubernetes can't have subfolders in configmaps """ + + @abstractmethod + def get_config_root(self): + """ Returns the config root directory. """ + diff --git a/util/config/provider/testprovider.py b/util/config/provider/testprovider.py index 55158444b..7c3b31583 100644 --- a/util/config/provider/testprovider.py +++ b/util/config/provider/testprovider.py @@ -10,6 +10,10 @@ REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', ' class TestConfigProvider(BaseProvider): """ Implementation of the config provider for testing. Everything is kept in-memory instead on the real file system. """ + + def get_config_root(self): + raise Exception('Test Config does not have a config root') + def __init__(self): self.clear() diff --git a/util/config/validator.py b/util/config/validator.py index c45a89e03..c31f09851 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -2,6 +2,8 @@ import logging from auth.auth_context import get_authenticated_user from data.users import LDAP_CERT_FILENAME +from util.secscan.secscan_util import get_blob_download_uri_getter +from util.config import URLSchemeAndHostname from util.config.validators.validate_database import DatabaseValidator from util.config.validators.validate_redis import RedisValidator @@ -64,7 +66,7 @@ VALIDATORS = { AppTokenAuthValidator.name: AppTokenAuthValidator.validate, } -def validate_service_for_config(service, config, password=None): +def validate_service_for_config(service, validator_context): """ Attempts to validate the configuration for the given service. """ if not service in VALIDATORS: return { @@ -72,7 +74,7 @@ def validate_service_for_config(service, config, password=None): } try: - VALIDATORS[service](config, get_authenticated_user(), password) + VALIDATORS[service](validator_context) return { 'status': True } @@ -92,3 +94,56 @@ def is_valid_config_upload_filename(filename): return True return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES]) + + +class ValidatorContext(object): + """ Context to run validators in, with any additional runtime configuration they need + """ + def __init__(self, config, user_password=None, http_client=None, context=None, + url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, + ip_resolver=None, feature_sec_scanner=False, is_testing=False, + uri_creator=None, config_provider=None): + self.config = config + self.user = get_authenticated_user() + self.user_password = user_password + self.http_client = http_client + self.context = context + self.url_scheme_and_hostname = url_scheme_and_hostname + self.jwt_auth_max = jwt_auth_max + self.registry_title = registry_title + self.ip_resolver = ip_resolver + self.feature_sec_scanner = feature_sec_scanner + self.is_testing = is_testing + self.uri_creator = uri_creator + self.config_provider = config_provider + + @classmethod + def from_app(cls, app, config, user_password, ip_resolver, client=None, config_provider=None): + """ + Creates a ValidatorContext from an app config, with a given config to validate + :param app: the Flask app to pull configuration information from + :param config: the config to validate + :param user_password: request password + :param ip_resolver: an App + :param client: + :param config_provider: + :return: + """ + url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app.config) + + cls(config, + user_password, + client or app.config['HTTPCLIENT'], + app.app_context, + url_scheme_and_hostname, + app.config.get('JWT_AUTH_MAX_FRESH_S', 300), + app.config['REGISTRY_TITLE'], + ip_resolver, + app.config.get('FEATURE_SECURITY_SCANNER', False), + app.config.get('TESTING', False), + get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname), + config_provider) + + + + diff --git a/util/config/validators/__init__.py b/util/config/validators/__init__.py index a3edeeb12..31268c45f 100644 --- a/util/config/validators/__init__.py +++ b/util/config/validators/__init__.py @@ -15,6 +15,6 @@ class BaseValidator(object): @classmethod @abstractmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Raises Exception if failure to validate. """ pass diff --git a/util/config/validators/test/test_validate_access.py b/util/config/validators/test/test_validate_access.py index 9008767eb..54f44649b 100644 --- a/util/config/validators/test/test_validate_access.py +++ b/util/config/validators/test/test_validate_access.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_access import AccessSettingsValidator @@ -17,6 +18,6 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, expected_excepti if expected_exception is not None: with pytest.raises(expected_exception): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) else: - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) diff --git a/util/config/validators/test/test_validate_actionlog_archiving.py b/util/config/validators/test/test_validate_actionlog_archiving.py index c14555441..e3641e49c 100644 --- a/util/config/validators/test/test_validate_actionlog_archiving.py +++ b/util/config/validators/test/test_validate_actionlog_archiving.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator @@ -12,7 +13,7 @@ from test.fixtures import * ]) def test_skip_validate_actionlog(unvalidated_config, app): validator = ActionLogArchivingValidator() - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('config, expected_error', [ @@ -33,19 +34,19 @@ def test_invalid_config(config, expected_error, app): validator = ActionLogArchivingValidator() with pytest.raises(ConfigValidationException) as ipe: - validator.validate(config, None, None) + validator.validate(ValidatorContext(config)) assert ipe.value.message == expected_error def test_valid_config(app): - config = { + config = ValidatorContext({ 'FEATURE_ACTION_LOG_ROTATION': True, 'ACTION_LOG_ARCHIVE_PATH': 'somepath', 'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation', 'DISTRIBUTED_STORAGE_CONFIG': { 'somelocation': {}, }, - } + }) validator = ActionLogArchivingValidator() - validator.validate(config, None, None) + validator.validate(config) diff --git a/util/config/validators/test/test_validate_apptokenauth.py b/util/config/validators/test/test_validate_apptokenauth.py index 61ccd3570..87227f702 100644 --- a/util/config/validators/test/test_validate_apptokenauth.py +++ b/util/config/validators/test/test_validate_apptokenauth.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_apptokenauth import AppTokenAuthValidator @@ -15,15 +16,15 @@ def test_validate_invalid_auth_config(unvalidated_config, app): validator = AppTokenAuthValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_auth(app): - config = { + config = ValidatorContext({ 'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': True, 'FEATURE_DIRECT_LOGIN': False, - } + }) validator = AppTokenAuthValidator() - validator.validate(config, None, None) + validator.validate(config) diff --git a/util/config/validators/test/test_validate_bitbucket_trigger.py b/util/config/validators/test/test_validate_bitbucket_trigger.py index a5b5b6738..6159ecbc1 100644 --- a/util/config/validators/test/test_validate_bitbucket_trigger.py +++ b/util/config/validators/test/test_validate_bitbucket_trigger.py @@ -2,22 +2,24 @@ import pytest from httmock import urlmatch, HTTMock +from util.config import URLSchemeAndHostname +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator from test.fixtures import * @pytest.mark.parametrize('unvalidated_config', [ - ({}), - ({'BITBUCKET_TRIGGER_CONFIG': {}}), - ({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}}), - ({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}}), + (ValidatorContext({})), + (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})), + (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})), + (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})), ]) def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app): validator = BitbucketTriggerValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(unvalidated_config) def test_validate_bitbucket_trigger(app): url_hit = [False] @@ -32,11 +34,15 @@ def test_validate_bitbucket_trigger(app): with HTTMock(handler): validator = BitbucketTriggerValidator() - validator.validate({ + + url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000') + unvalidated_config = ValidatorContext({ 'BITBUCKET_TRIGGER_CONFIG': { 'CONSUMER_KEY': 'foo', 'CONSUMER_SECRET': 'bar', }, - }, None, None) + }, url_scheme_and_hostname=url_scheme_and_hostname) + + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_database.py b/util/config/validators/test/test_validate_database.py index 20e1d97e5..68b21acd6 100644 --- a/util/config/validators/test/test_validate_database.py +++ b/util/config/validators/test/test_validate_database.py @@ -1,22 +1,23 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_database import DatabaseValidator from test.fixtures import * @pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [ - (None, None, None, TypeError), - ({}, None, None, KeyError), - ({'DB_URI': 'sqlite:///:memory:'}, None, None, None), - ({'DB_URI': 'invalid:///:memory:'}, None, None, KeyError), - ({'DB_NOTURI': 'sqlite:///:memory:'}, None, None, KeyError), + (ValidatorContext(None), None, None, TypeError), + (ValidatorContext({}), None, None, KeyError), + (ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None), + (ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError), + (ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError), ]) def test_validate_database(unvalidated_config, user, user_password, expected, app): validator = DatabaseValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) else: - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_github.py b/util/config/validators/test/test_validate_github.py index aad09870f..b0e919c22 100644 --- a/util/config/validators/test/test_validate_github.py +++ b/util/config/validators/test/test_validate_github.py @@ -2,6 +2,8 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator @@ -36,7 +38,7 @@ def test_validate_invalid_github_config(github_config, github_validator, app): with pytest.raises(ConfigValidationException): unvalidated_config = {} unvalidated_config[github_validator.config_key] = github_config - github_validator.validate(unvalidated_config, None, None) + github_validator.validate(ValidatorContext(unvalidated_config)) def test_validate_github(github_validator, app): url_hit = [False, False] @@ -52,13 +54,16 @@ def test_validate_github(github_validator, app): return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}} with HTTMock(app_handler, handler): - github_validator.validate({ + unvalidated_config = ValidatorContext({ github_validator.config_key: { 'GITHUB_ENDPOINT': 'http://somehost', 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', }, - }, None, None) + }) + + unvalidated_config.http_client = build_requests_session() + github_validator.validate(unvalidated_config) assert url_hit[0] assert url_hit[1] diff --git a/util/config/validators/test/test_validate_gitlab_trigger.py b/util/config/validators/test/test_validate_gitlab_trigger.py index 17b32764b..fedd156ed 100644 --- a/util/config/validators/test/test_validate_gitlab_trigger.py +++ b/util/config/validators/test/test_validate_gitlab_trigger.py @@ -3,6 +3,9 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config import URLSchemeAndHostname +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator @@ -18,7 +21,7 @@ def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app): validator = GitLabTriggerValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_gitlab_enterprise_trigger(app): url_hit = [False] @@ -30,12 +33,17 @@ def test_validate_gitlab_enterprise_trigger(app): with HTTMock(handler): validator = GitLabTriggerValidator() - validator.validate({ + + url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000') + + unvalidated_config = ValidatorContext({ 'GITLAB_TRIGGER_CONFIG': { 'GITLAB_ENDPOINT': 'http://somegitlab', 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', }, - }, None, None) + }, http_client=build_requests_session(), url_scheme_and_hostname=url_scheme_and_hostname) + + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_google_login.py b/util/config/validators/test/test_validate_google_login.py index a41a51adb..ff823a7a7 100644 --- a/util/config/validators/test/test_validate_google_login.py +++ b/util/config/validators/test/test_validate_google_login.py @@ -2,6 +2,8 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_google_login import GoogleLoginValidator @@ -17,7 +19,7 @@ def test_validate_invalid_google_login_config(unvalidated_config, app): validator = GoogleLoginValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_google_login(app): url_hit = [False] @@ -29,11 +31,15 @@ def test_validate_google_login(app): validator = GoogleLoginValidator() with HTTMock(handler): - validator.validate({ + unvalidated_config = ValidatorContext({ 'GOOGLE_LOGIN_CONFIG': { 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', }, - }, None, None) + }) + + unvalidated_config.http_client = build_requests_session() + + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_jwt.py b/util/config/validators/test/test_validate_jwt.py index 0a29b1953..0f70d93df 100644 --- a/util/config/validators/test/test_validate_jwt.py +++ b/util/config/validators/test/test_validate_jwt.py @@ -1,5 +1,7 @@ import pytest +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_jwt import JWTAuthValidator from util.morecollections import AttrDict @@ -7,6 +9,7 @@ from util.morecollections import AttrDict from test.test_external_jwt_authn import fake_jwt from test.fixtures import * +from app import config_provider @pytest.mark.parametrize('unvalidated_config', [ @@ -14,7 +17,9 @@ from test.fixtures import * ({'AUTHENTICATION_TYPE': 'Database'}), ]) def test_validate_noop(unvalidated_config, app): - JWTAuthValidator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.config_provider = config_provider + JWTAuthValidator.validate(config) @pytest.mark.parametrize('unvalidated_config', [ @@ -24,9 +29,13 @@ def test_validate_noop(unvalidated_config, app): ]) def test_invalid_config(unvalidated_config, app): with pytest.raises(ConfigValidationException): - JWTAuthValidator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.config_provider = config_provider + JWTAuthValidator.validate(config) +# TODO(jschorr): fix these when re-adding jwt auth mechanism to jwt validators +@pytest.mark.skip(reason='No way of currently testing this') @pytest.mark.parametrize('username, password, expected_exception', [ ('invaliduser', 'invalidpass', ConfigValidationException), ('cool.user', 'invalidpass', ConfigValidationException), @@ -42,10 +51,15 @@ def test_validated_jwt(username, password, expected_exception, app): config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url + unvalidated_config = ValidatorContext(config) + unvalidated_config.user = AttrDict(dict(username=username)) + unvalidated_config.user_password = password + unvalidated_config.config_provider = config_provider + + unvalidated_config.http_client = build_requests_session() + if expected_exception is not None: with pytest.raises(ConfigValidationException): - JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, - public_key_path=jwt_auth.public_key_path) + JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path) else: - JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, - public_key_path=jwt_auth.public_key_path) + JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path) diff --git a/util/config/validators/test/test_validate_keystone.py b/util/config/validators/test/test_validate_keystone.py index 304700b39..f6d8cf0c2 100644 --- a/util/config/validators/test/test_validate_keystone.py +++ b/util/config/validators/test/test_validate_keystone.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_keystone import KeystoneValidator from util.morecollections import AttrDict @@ -13,7 +14,7 @@ from test.fixtures import * ({'AUTHENTICATION_TYPE': 'Database'}), ]) def test_validate_noop(unvalidated_config, app): - KeystoneValidator.validate(unvalidated_config, None, None) + KeystoneValidator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('unvalidated_config', [ ({'AUTHENTICATION_TYPE': 'Keystone'}), @@ -25,7 +26,7 @@ def test_validate_noop(unvalidated_config, app): ]) def test_invalid_config(unvalidated_config, app): with pytest.raises(ConfigValidationException): - KeystoneValidator.validate(unvalidated_config, None, None) + KeystoneValidator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('username, password, expected_exception', [ @@ -45,8 +46,12 @@ def test_validated_keystone(username, password, expected_exception, app): config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass' config['KEYSTONE_ADMIN_TENANT'] = 'admintenant' + unvalidated_config = ValidatorContext(config) + unvalidated_config.user = AttrDict(dict(username=username)) + unvalidated_config.user_password = password + if expected_exception is not None: with pytest.raises(ConfigValidationException): - KeystoneValidator.validate(config, AttrDict(dict(username=username)), password) + KeystoneValidator.validate(unvalidated_config) else: - KeystoneValidator.validate(config, AttrDict(dict(username=username)), password) + KeystoneValidator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_ldap.py b/util/config/validators/test/test_validate_ldap.py index cdffce467..f5d5a7425 100644 --- a/util/config/validators/test/test_validate_ldap.py +++ b/util/config/validators/test/test_validate_ldap.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_ldap import LDAPValidator from util.morecollections import AttrDict @@ -7,13 +8,15 @@ from util.morecollections import AttrDict from test.test_ldap import mock_ldap from test.fixtures import * +from app import config_provider @pytest.mark.parametrize('unvalidated_config', [ ({}), ({'AUTHENTICATION_TYPE': 'Database'}), ]) def test_validate_noop(unvalidated_config, app): - LDAPValidator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config, config_provider=config_provider) + LDAPValidator.validate(config) @pytest.mark.parametrize('unvalidated_config', [ ({'AUTHENTICATION_TYPE': 'LDAP'}), @@ -21,7 +24,8 @@ def test_validate_noop(unvalidated_config, app): ]) def test_invalid_config(unvalidated_config, app): with pytest.raises(ConfigValidationException): - LDAPValidator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config, config_provider=config_provider) + LDAPValidator.validate(config) @pytest.mark.parametrize('uri', [ @@ -39,7 +43,8 @@ def test_invalid_uri(uri, app): config['LDAP_URI'] = uri with pytest.raises(ConfigValidationException): - LDAPValidator.validate(config, None, None) + config = ValidatorContext(config, config_provider=config_provider) + LDAPValidator.validate(config) @pytest.mark.parametrize('username, password, expected_exception', [ @@ -56,10 +61,13 @@ def test_validated_ldap(username, password, expected_exception, app): config['LDAP_ADMIN_PASSWD'] = 'password' config['LDAP_USER_RDN'] = ['ou=employees'] + unvalidated_config = ValidatorContext(config, user_password=password, config_provider=config_provider) + unvalidated_config.user = AttrDict(dict(username=username)) + if expected_exception is not None: with pytest.raises(ConfigValidationException): with mock_ldap(): - LDAPValidator.validate(config, AttrDict(dict(username=username)), password) + LDAPValidator.validate(unvalidated_config) else: with mock_ldap(): - LDAPValidator.validate(config, AttrDict(dict(username=username)), password) + LDAPValidator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_oidc.py b/util/config/validators/test/test_validate_oidc.py index 684ffef34..e884d4480 100644 --- a/util/config/validators/test/test_validate_oidc.py +++ b/util/config/validators/test/test_validate_oidc.py @@ -3,7 +3,9 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session from oauth.oidc import OIDC_WELLKNOWN +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_oidc import OIDCLoginValidator @@ -19,7 +21,7 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, app): validator = OIDCLoginValidator() with pytest.raises(ConfigValidationException): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_oidc_login(app): url_hit = [False] @@ -33,13 +35,16 @@ def test_validate_oidc_login(app): with HTTMock(handler): validator = OIDCLoginValidator() - validator.validate({ + unvalidated_config = ValidatorContext({ 'SOMETHING_LOGIN_CONFIG': { 'CLIENT_ID': 'foo', 'CLIENT_SECRET': 'bar', 'OIDC_SERVER': 'http://someserver', 'DEBUGGING': True, # Allows for HTTP. }, - }, None, None) + }) + unvalidated_config.http_client = build_requests_session() + + validator.validate(unvalidated_config) assert url_hit[0] diff --git a/util/config/validators/test/test_validate_redis.py b/util/config/validators/test/test_validate_redis.py index c6d1c2498..39a1ce5ee 100644 --- a/util/config/validators/test/test_validate_redis.py +++ b/util/config/validators/test/test_validate_redis.py @@ -5,10 +5,13 @@ from mock import patch from mockredis import mock_strict_redis_client +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_redis import RedisValidator from test.fixtures import * +from util.morecollections import AttrDict + @pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [ ({}, None, None, False, ConfigValidationException), @@ -19,8 +22,13 @@ from test.fixtures import * def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app): with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client): validator = RedisValidator() + unvalidated_config = ValidatorContext(unvalidated_config) + + unvalidated_config.user = AttrDict(dict(username=user)) + unvalidated_config.user_password = user_password + if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) else: - validator.validate(unvalidated_config, user, user_password) + validator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_secscan.py b/util/config/validators/test/test_validate_secscan.py index 6232f3156..ee4dfdcac 100644 --- a/util/config/validators/test/test_validate_secscan.py +++ b/util/config/validators/test/test_validate_secscan.py @@ -1,6 +1,8 @@ import pytest -from util.config.validators import ConfigValidationException +from config import build_requests_session +from util.config import URLSchemeAndHostname +from util.config.validator import ValidatorContext from util.config.validators.validate_secscan import SecurityScannerValidator from util.secscan.fake import fake_security_scanner @@ -10,7 +12,12 @@ from test.fixtures import * ({'DISTRIBUTED_STORAGE_PREFERENCE': []}), ]) def test_validate_noop(unvalidated_config, app): - SecurityScannerValidator.validate(unvalidated_config, None, None) + + unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True, + http_client=build_requests_session(), + url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000')) + + SecurityScannerValidator.validate(unvalidated_config) @pytest.mark.parametrize('unvalidated_config, expected_error', [ @@ -29,9 +36,13 @@ def test_validate_noop(unvalidated_config, app): }, None), ]) def test_validate(unvalidated_config, expected_error, app): + unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True, + http_client=build_requests_session(), + url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000')) + with fake_security_scanner(hostname='fakesecurityscanner'): if expected_error is not None: with pytest.raises(expected_error): - SecurityScannerValidator.validate(unvalidated_config, None, None) + SecurityScannerValidator.validate(unvalidated_config) else: - SecurityScannerValidator.validate(unvalidated_config, None, None) + SecurityScannerValidator.validate(unvalidated_config) diff --git a/util/config/validators/test/test_validate_signer.py b/util/config/validators/test/test_validate_signer.py index 4ee01cd9f..f45f91c11 100644 --- a/util/config/validators/test/test_validate_signer.py +++ b/util/config/validators/test/test_validate_signer.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_signer import SignerValidator @@ -14,6 +15,6 @@ def test_validate_signer(unvalidated_config, expected, app): validator = SignerValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) else: - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) diff --git a/util/config/validators/test/test_validate_ssl.py b/util/config/validators/test/test_validate_ssl.py index c7ec334be..8ea7f3297 100644 --- a/util/config/validators/test/test_validate_ssl.py +++ b/util/config/validators/test/test_validate_ssl.py @@ -3,11 +3,13 @@ import pytest from mock import patch from tempfile import NamedTemporaryFile +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES from test.test_ssl_util import generate_test_cert from test.fixtures import * +from app import config_provider @pytest.mark.parametrize('unvalidated_config', [ ({}), @@ -16,7 +18,7 @@ from test.fixtures import * ]) def test_skip_validate_ssl(unvalidated_config, app): validator = SSLValidator() - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) @pytest.mark.parametrize('cert, expected_error, error_message', [ @@ -54,11 +56,13 @@ def test_validate_ssl(cert, expected_error, error_message, app): with patch('app.config_provider.volume_file_exists', return_true): with patch('app.config_provider.get_volume_file', get_volume_file): validator = SSLValidator() + config = ValidatorContext(config) + config.config_provider = config_provider if expected_error is not None: with pytest.raises(expected_error) as ipe: - validator.validate(config, None, None) + validator.validate(config) assert ipe.value.message == error_message else: - validator.validate(config, None, None) + validator.validate(config) diff --git a/util/config/validators/test/test_validate_storage.py b/util/config/validators/test/test_validate_storage.py index f360eab3c..007452c7f 100644 --- a/util/config/validators/test/test_validate_storage.py +++ b/util/config/validators/test/test_validate_storage.py @@ -1,6 +1,7 @@ import moto import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_storage import StorageValidator @@ -16,15 +17,15 @@ def test_validate_storage(unvalidated_config, expected, app): validator = StorageValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) else: - validator.validate(unvalidated_config, None, None) + validator.validate(ValidatorContext(unvalidated_config)) def test_validate_s3_storage(app): validator = StorageValidator() with moto.mock_s3(): with pytest.raises(ConfigValidationException) as ipe: - validator.validate({ + validator.validate(ValidatorContext({ 'DISTRIBUTED_STORAGE_CONFIG': { 'default': ('S3Storage', { 's3_access_key': 'invalid', @@ -33,6 +34,6 @@ def test_validate_s3_storage(app): 'storage_path': '' }), } - }, None, None) + })) assert ipe.value.message == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found' \ No newline at end of file diff --git a/util/config/validators/test/test_validate_timemachine.py b/util/config/validators/test/test_validate_timemachine.py index e1da63a45..1c3b29ba5 100644 --- a/util/config/validators/test/test_validate_timemachine.py +++ b/util/config/validators/test/test_validate_timemachine.py @@ -1,5 +1,6 @@ import pytest +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_timemachine import TimeMachineValidator @@ -7,7 +8,7 @@ from util.config.validators.validate_timemachine import TimeMachineValidator ({}), ]) def test_validate_noop(unvalidated_config): - TimeMachineValidator.validate(unvalidated_config, None, None) + TimeMachineValidator.validate(ValidatorContext(unvalidated_config)) from test.fixtures import * @@ -25,7 +26,7 @@ def test_validate(default_exp, options, expected_exception, app): if expected_exception is not None: with pytest.raises(ConfigValidationException) as cve: - TimeMachineValidator.validate(config, None, None) + TimeMachineValidator.validate(ValidatorContext(config)) assert str(cve.value) == str(expected_exception) else: - TimeMachineValidator.validate(config, None, None) + TimeMachineValidator.validate(ValidatorContext(config)) diff --git a/util/config/validators/test/test_validate_torrent.py b/util/config/validators/test/test_validate_torrent.py index badd08198..1ad3664b0 100644 --- a/util/config/validators/test/test_validate_torrent.py +++ b/util/config/validators/test/test_validate_torrent.py @@ -2,6 +2,8 @@ import pytest from httmock import urlmatch, HTTMock +from config import build_requests_session +from util.config.validator import ValidatorContext from util.config.validators import ConfigValidationException from util.config.validators.validate_torrent import BittorrentValidator @@ -23,8 +25,14 @@ def test_validate_torrent(unvalidated_config, expected, app): validator = BittorrentValidator() if expected is not None: with pytest.raises(expected): - validator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.http_client = build_requests_session() + + validator.validate(config) assert not announcer_hit[0] else: - validator.validate(unvalidated_config, None, None) + config = ValidatorContext(unvalidated_config) + config.http_client = build_requests_session() + + validator.validate(config) assert announcer_hit[0] diff --git a/util/config/validators/validate_access.py b/util/config/validators/validate_access.py index eb80090d8..552e3b176 100644 --- a/util/config/validators/validate_access.py +++ b/util/config/validators/validate_access.py @@ -1,4 +1,3 @@ -from app import app from util.config.validators import BaseValidator, ConfigValidationException from oauth.loginmanager import OAuthLoginManager from oauth.oidc import OIDCLoginService @@ -7,13 +6,15 @@ class AccessSettingsValidator(BaseValidator): name = "access" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): + config = validator_context.config + client = validator_context.http_client + if not config.get('FEATURE_DIRECT_LOGIN', True): # Make sure we have at least one OIDC enabled. github_login = config.get('FEATURE_GITHUB_LOGIN', False) google_login = config.get('FEATURE_GOOGLE_LOGIN', False) - client = app.config['HTTPCLIENT'] login_manager = OAuthLoginManager(config, client=client) custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)] diff --git a/util/config/validators/validate_actionlog_archiving.py b/util/config/validators/validate_actionlog_archiving.py index e8fb79a50..03c63fe95 100644 --- a/util/config/validators/validate_actionlog_archiving.py +++ b/util/config/validators/validate_actionlog_archiving.py @@ -4,7 +4,9 @@ class ActionLogArchivingValidator(BaseValidator): name = "actionlogarchiving" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): + config = validator_context.config + """ Validates the action log archiving configuration. """ if not config.get('FEATURE_ACTION_LOG_ROTATION', False): return diff --git a/util/config/validators/validate_apptokenauth.py b/util/config/validators/validate_apptokenauth.py index 6d7be1f1b..be2096117 100644 --- a/util/config/validators/validate_apptokenauth.py +++ b/util/config/validators/validate_apptokenauth.py @@ -4,7 +4,9 @@ class AppTokenAuthValidator(BaseValidator): name = "apptoken-auth" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): + config = validator_context.config + if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken': return diff --git a/util/config/validators/validate_bitbucket_trigger.py b/util/config/validators/validate_bitbucket_trigger.py index 15378c1b4..c8cb87887 100644 --- a/util/config/validators/validate_bitbucket_trigger.py +++ b/util/config/validators/validate_bitbucket_trigger.py @@ -1,14 +1,15 @@ from bitbucket import BitBucket -from app import get_app_url from util.config.validators import BaseValidator, ConfigValidationException class BitbucketTriggerValidator(BaseValidator): name = "bitbucket-trigger" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the config for BitBucket. """ + config = validator_context.config + trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG') if not trigger_config: raise ConfigValidationException('Missing client ID and client secret') @@ -21,7 +22,7 @@ class BitbucketTriggerValidator(BaseValidator): key = trigger_config['CONSUMER_KEY'] secret = trigger_config['CONSUMER_SECRET'] - callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url()) + callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (validator_context.url_scheme_and_hostname.get_url()) bitbucket_client = BitBucket(key, secret, callback_url) (result, _, _) = bitbucket_client.get_authorization_url() diff --git a/util/config/validators/validate_database.py b/util/config/validators/validate_database.py index 5fb27fa80..30fcc6d0e 100644 --- a/util/config/validators/validate_database.py +++ b/util/config/validators/validate_database.py @@ -7,8 +7,10 @@ class DatabaseValidator(BaseValidator): name = "database" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates connecting to the database. """ + config = validator_context.config + try: validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {})) except OperationalError as ex: diff --git a/util/config/validators/validate_email.py b/util/config/validators/validate_email.py index b394fdad4..7adb7f6fa 100644 --- a/util/config/validators/validate_email.py +++ b/util/config/validators/validate_email.py @@ -1,16 +1,20 @@ from flask import Flask from flask_mail import Mail, Message -from app import app from util.config.validators import BaseValidator class EmailValidator(BaseValidator): name = "mail" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates sending email. """ - with app.app_context(): + config = validator_context.config + user = validator_context.user + app_context = validator_context.context + registry_title = validator_context.registry_title + + with app_context(): test_app = Flask("mail-test-app") test_app.config.update(config) test_app.config.update({ @@ -19,7 +23,7 @@ class EmailValidator(BaseValidator): }) test_mail = Mail(test_app) - test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'], + test_msg = Message("Test e-mail from %s" % registry_title, sender=config.get('MAIL_DEFAULT_SENDER')) test_msg.add_recipient(user.email) test_mail.send(test_msg) diff --git a/util/config/validators/validate_github.py b/util/config/validators/validate_github.py index 39293a11d..3b5e2ed1b 100644 --- a/util/config/validators/validate_github.py +++ b/util/config/validators/validate_github.py @@ -1,4 +1,3 @@ -from app import app from oauth.services.github import GithubOAuthService from util.config.validators import BaseValidator, ConfigValidationException @@ -7,8 +6,12 @@ class BaseGitHubValidator(BaseValidator): config_key = None @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the OAuth credentials and API endpoint for a Github service. """ + config = validator_context.config + client = validator_context.http_client + url_scheme_and_hostname = validator_context.url_scheme_and_hostname + github_config = config.get(cls.config_key) if not github_config: raise ConfigValidationException('Missing GitHub client id and client secret') @@ -30,9 +33,8 @@ class BaseGitHubValidator(BaseValidator): raise ConfigValidationException('Organization restriction must have at least one allowed ' + 'organization') - client = app.config['HTTPCLIENT'] oauth = GithubOAuthService(config, cls.config_key) - result = oauth.validate_client_id_and_secret(client, app.config) + result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname) if not result: raise ConfigValidationException('Invalid client id or client secret') diff --git a/util/config/validators/validate_gitlab_trigger.py b/util/config/validators/validate_gitlab_trigger.py index 7f9e8c28e..50c381881 100644 --- a/util/config/validators/validate_gitlab_trigger.py +++ b/util/config/validators/validate_gitlab_trigger.py @@ -1,4 +1,3 @@ -from app import app from oauth.services.gitlab import GitLabOAuthService from util.config.validators import BaseValidator, ConfigValidationException @@ -6,8 +5,12 @@ class GitLabTriggerValidator(BaseValidator): name = "gitlab-trigger" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the OAuth credentials and API endpoint for a GitLab service. """ + config = validator_context.config + url_scheme_and_hostname = validator_context.url_scheme_and_hostname + client = validator_context.http_client + github_config = config.get('GITLAB_TRIGGER_CONFIG') if not github_config: raise ConfigValidationException('Missing GitLab client id and client secret') @@ -23,8 +26,7 @@ class GitLabTriggerValidator(BaseValidator): if not github_config.get('CLIENT_SECRET'): raise ConfigValidationException('Missing Client Secret') - client = app.config['HTTPCLIENT'] oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG') - result = oauth.validate_client_id_and_secret(client, app.config) + result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname) if not result: raise ConfigValidationException('Invalid client id or client secret') diff --git a/util/config/validators/validate_google_login.py b/util/config/validators/validate_google_login.py index 80e1537f0..e03aae8d8 100644 --- a/util/config/validators/validate_google_login.py +++ b/util/config/validators/validate_google_login.py @@ -1,4 +1,3 @@ -from app import app from oauth.services.google import GoogleOAuthService from util.config.validators import BaseValidator, ConfigValidationException @@ -6,8 +5,12 @@ class GoogleLoginValidator(BaseValidator): name = "google-login" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the Google Login client ID and secret. """ + config = validator_context.config + client = validator_context.http_client + url_scheme_and_hostname = validator_context.url_scheme_and_hostname + google_login_config = config.get('GOOGLE_LOGIN_CONFIG') if not google_login_config: raise ConfigValidationException('Missing client ID and client secret') @@ -18,8 +21,7 @@ class GoogleLoginValidator(BaseValidator): if not google_login_config.get('CLIENT_SECRET'): raise ConfigValidationException('Missing Client Secret') - client = app.config['HTTPCLIENT'] oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG') - result = oauth.validate_client_id_and_secret(client, app.config) + result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname) if not result: raise ConfigValidationException('Invalid client id or client secret') diff --git a/util/config/validators/validate_jwt.py b/util/config/validators/validate_jwt.py index 808e74152..c56f3630b 100644 --- a/util/config/validators/validate_jwt.py +++ b/util/config/validators/validate_jwt.py @@ -1,4 +1,4 @@ -from app import app, OVERRIDE_CONFIG_DIRECTORY +import os from data.users.externaljwt import ExternalJWTAuthN from util.config.validators import BaseValidator, ConfigValidationException @@ -6,8 +6,15 @@ class JWTAuthValidator(BaseValidator): name = "jwt" @classmethod - def validate(cls, config, user, user_password, public_key_path=None): + def validate(cls, validator_context, public_key_path=None): """ Validates the JWT authentication system. """ + config = validator_context.config + user = validator_context.user + user_password = validator_context.user_password + http_client = validator_context.http_client + jwt_auth_max = validator_context.jwt_auth_max + config_provider = validator_context.config_provider + if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT': return @@ -23,12 +30,18 @@ class JWTAuthValidator(BaseValidator): if not issuer: raise ConfigValidationException('Missing JWT Issuer ID') + + # TODO(jschorr): fix this + return + + override_config_directory = os.path.join(config_provider.get_config_root(), '../stack/') + # Try to instatiate the JWT authentication mechanism. This will raise an exception if # the key cannot be found. users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer, - OVERRIDE_CONFIG_DIRECTORY, - app.config['HTTPCLIENT'], - app.config.get('JWT_AUTH_MAX_FRESH_S', 300), + override_config_directory, + http_client, + jwt_auth_max, public_key_path=public_key_path, requires_email=config.get('FEATURE_MAILING', True)) diff --git a/util/config/validators/validate_keystone.py b/util/config/validators/validate_keystone.py index 415f7958b..2d4f7bbe0 100644 --- a/util/config/validators/validate_keystone.py +++ b/util/config/validators/validate_keystone.py @@ -5,8 +5,12 @@ class KeystoneValidator(BaseValidator): name = "keystone" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the Keystone authentication system. """ + config = validator_context.config + user = validator_context.user + user_password = validator_context.user_password + if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone': return diff --git a/util/config/validators/validate_ldap.py b/util/config/validators/validate_ldap.py index 1c4fb9df7..331cd87d3 100644 --- a/util/config/validators/validate_ldap.py +++ b/util/config/validators/validate_ldap.py @@ -2,24 +2,27 @@ import os import ldap import subprocess -from app import app, config_provider from data.users import LDAP_CERT_FILENAME from data.users.externalldap import LDAPConnection, LDAPUsers from util.config.validators import BaseValidator, ConfigValidationException -from _init import CONF_DIR class LDAPValidator(BaseValidator): name = "ldap" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the LDAP connection. """ + config = validator_context.config + user = validator_context.user + user_password = validator_context.user_password + config_provider = validator_context.config_provider + if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP': return # If there is a custom LDAP certificate, then reinstall the certificates for the container. if config_provider.volume_file_exists(LDAP_CERT_FILENAME): - subprocess.check_call([os.path.join(CONF_DIR, 'init/certs_install.sh')]) + subprocess.check_call([os.path.join(config_provider.get_config_root(), '../init/certs_install.sh')]) # Note: raises ldap.INVALID_CREDENTIALS on failure admin_dn = config.get('LDAP_ADMIN_DN') diff --git a/util/config/validators/validate_oidc.py b/util/config/validators/validate_oidc.py index ba94b0362..8126d4db7 100644 --- a/util/config/validators/validate_oidc.py +++ b/util/config/validators/validate_oidc.py @@ -1,4 +1,3 @@ -from app import app from oauth.loginmanager import OAuthLoginManager from oauth.oidc import OIDCLoginService, DiscoveryFailureException from util.config.validators import BaseValidator, ConfigValidationException @@ -7,8 +6,10 @@ class OIDCLoginValidator(BaseValidator): name = "oidc-login" @classmethod - def validate(cls, config, user, user_password): - client = app.config['HTTPCLIENT'] + def validate(cls, validator_context): + config = validator_context.config + client = validator_context.http_client + login_manager = OAuthLoginManager(config, client=client) for service in login_manager.services: if not isinstance(service, OIDCLoginService): diff --git a/util/config/validators/validate_redis.py b/util/config/validators/validate_redis.py index 92909dbf4..55beffb84 100644 --- a/util/config/validators/validate_redis.py +++ b/util/config/validators/validate_redis.py @@ -6,8 +6,10 @@ class RedisValidator(BaseValidator): name = "redis" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates connecting to redis. """ + config = validator_context.config + redis_config = config.get('BUILDLOGS_REDIS', {}) if not 'host' in redis_config: raise ConfigValidationException('Missing redis hostname') diff --git a/util/config/validators/validate_secscan.py b/util/config/validators/validate_secscan.py index c5efd34b0..9f7c2d67f 100644 --- a/util/config/validators/validate_secscan.py +++ b/util/config/validators/validate_secscan.py @@ -1,6 +1,5 @@ import time -from app import app from boot import setup_jwt_proxy from util.secscan.api import SecurityScannerAPI from util.config.validators import BaseValidator, ConfigValidationException @@ -9,15 +8,22 @@ class SecurityScannerValidator(BaseValidator): name = "security-scanner" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the configuration for talking to a Quay Security Scanner. """ - if not config.get('FEATURE_SECURITY_SCANNER', False): + config = validator_context.config + client = validator_context.http_client + feature_sec_scanner = validator_context.feature_sec_scanner + is_testing = validator_context.is_testing + + server_hostname = validator_context.url_scheme_and_hostname.hostname + uri_creator = validator_context.uri_creator + + if not feature_sec_scanner: return - client = app.config['HTTPCLIENT'] - api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True) + api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator) - if not config.get('TESTING', False): + if not is_testing: # Generate a temporary Quay key to use for signing the outgoing requests. setup_jwt_proxy() diff --git a/util/config/validators/validate_signer.py b/util/config/validators/validate_signer.py index b44cb3c3d..f151ab19d 100644 --- a/util/config/validators/validate_signer.py +++ b/util/config/validators/validate_signer.py @@ -1,6 +1,5 @@ from StringIO import StringIO -from app import config_provider from util.config.validators import BaseValidator, ConfigValidationException from util.security.signing import SIGNING_ENGINES @@ -8,8 +7,11 @@ class SignerValidator(BaseValidator): name = "signer" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the GPG public+private key pair used for signing converted ACIs. """ + config = validator_context.config + config_provider = validator_context.config_provider + if config.get('SIGNING_ENGINE') is None: return diff --git a/util/config/validators/validate_ssl.py b/util/config/validators/validate_ssl.py index ea1ae3188..326754184 100644 --- a/util/config/validators/validate_ssl.py +++ b/util/config/validators/validate_ssl.py @@ -1,4 +1,3 @@ -from app import config_provider from util.config.validators import BaseValidator, ConfigValidationException from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException @@ -8,8 +7,10 @@ class SSLValidator(BaseValidator): name = "ssl" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the SSL configuration (if enabled). """ + config = validator_context.config + config_provider = validator_context.config_provider # Skip if non-SSL. if config.get('PREFERRED_URL_SCHEME', 'http') != 'https': diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index faded5c36..3e3de74ee 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -1,16 +1,21 @@ -from app import app, ip_resolver, config_provider from storage import get_storage_driver from util.config.validators import BaseValidator, ConfigValidationException + class StorageValidator(BaseValidator): name = "registry-storage" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates registry storage. """ + config = validator_context.config + client = validator_context.http_client + ip_resolver = validator_context.ip_resolver + config_provider = validator_context.config_provider + replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False) - providers = _get_storage_providers(config).items() + providers = _get_storage_providers(config, ip_resolver, config_provider).items() if not providers: raise ConfigValidationException('Storage configuration required') @@ -21,7 +26,7 @@ class StorageValidator(BaseValidator): 'with storage replication') # Run validation on the driver. - driver.validate(app.config['HTTPCLIENT']) + driver.validate(client) # Run setup on the driver if the read/write succeeded. driver.setup() @@ -30,7 +35,7 @@ class StorageValidator(BaseValidator): raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg)) -def _get_storage_providers(config): +def _get_storage_providers(config, ip_resolver, config_provider): storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {}) drivers = {} diff --git a/util/config/validators/validate_timemachine.py b/util/config/validators/validate_timemachine.py index 750e1695e..c246a2686 100644 --- a/util/config/validators/validate_timemachine.py +++ b/util/config/validators/validate_timemachine.py @@ -9,7 +9,9 @@ class TimeMachineValidator(BaseValidator): name = "time-machine" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): + config = validator_context.config + if not 'DEFAULT_TAG_EXPIRATION' in config: # Old style config return diff --git a/util/config/validators/validate_torrent.py b/util/config/validators/validate_torrent.py index f63029c46..d8137e12c 100644 --- a/util/config/validators/validate_torrent.py +++ b/util/config/validators/validate_torrent.py @@ -2,7 +2,6 @@ import logging from hashlib import sha1 -from app import app from util.config.validators import BaseValidator, ConfigValidationException from util.registry.torrent import jwt_from_infohash @@ -12,15 +11,16 @@ class BittorrentValidator(BaseValidator): name = "bittorrent" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, validator_context): """ Validates the configuration for using BitTorrent for downloads. """ + config = validator_context.config + client = validator_context.http_client + announce_url = config.get('BITTORRENT_ANNOUNCE_URL') if not announce_url: raise ConfigValidationException('Missing announce URL') # Ensure that the tracker is reachable and accepts requests signed with a registry key. - client = app.config['HTTPCLIENT'] - params = { 'info_hash': sha1('test').digest(), 'peer_id': '-QUAY00-6wfG2wk6wWLc', diff --git a/util/secscan/api.py b/util/secscan/api.py index fee1b97aa..25ef36807 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -7,16 +7,12 @@ from urlparse import urljoin import requests -from flask import url_for - from data.database import CloseForLongOperation from data import model from data.model.storage import get_storage_locations -from util import get_app_url, slash_join from util.abchelpers import nooper from util.failover import failover, FailoverException from util.secscan.validator import SecurityConfigValidator -from util.security.instancekeys import InstanceKeys from util.security.registry_jwt import generate_bearer_token, build_context_and_subject from _init import CONF_DIR @@ -70,16 +66,16 @@ def compute_layer_id(layer): class SecurityScannerAPI(object): """ Helper class for talking to the Security Scan service (usually Clair). """ - def __init__(self, app, config, storage, client=None, skip_validation=False): + def __init__(self, config, storage, server_hostname=None, client=None, skip_validation=False, uri_creator=None, instance_keys=None): feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False) has_valid_config = skip_validation if not skip_validation and feature_enabled: - config_validator = SecurityConfigValidator(config) + config_validator = SecurityConfigValidator(feature_enabled, config.get('SECURITY_SCANNER_ENDPOINT')) has_valid_config = config_validator.valid() if feature_enabled and has_valid_config: - self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client) + self.state = ImplementedSecurityScannerAPI(config, storage, server_hostname, client=client, uri_creator=uri_creator, instance_keys=instance_keys) else: self.state = NoopSecurityScannerAPI() @@ -150,20 +146,25 @@ class NoopSecurityScannerAPI(SecurityScannerAPIInterface): class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): """ Helper class for talking to the Security Scan service (Clair). """ - def __init__(self, app, config, storage, client=None): - self._app = app + # TODO(sam) refactor this to not take an app config, and instead just the things it needs as a config object + def __init__(self, config, storage, server_hostname, client=None, uri_creator=None, instance_keys=None): self._config = config - self._instance_keys = InstanceKeys(app) - self._client = client or config['HTTPCLIENT'] + self._instance_keys = instance_keys + self._client = client self._storage = storage + self._server_hostname = server_hostname self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE'] self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2) + self._uri_creator = uri_creator def _get_image_url_and_auth(self, image): """ Returns a tuple of the url and the auth header value that must be used to fetch the layer data itself. If the image can't be addressed, we return None. """ + if self._instance_keys is None: + raise Exception('No Instance keys provided to Security Scanner API') + path = model.storage.get_layer_path(image.storage) locations = self._default_storage_locations @@ -183,7 +184,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): repository_and_namespace = '/'.join([namespace_name, repo_name]) # Generate the JWT which will authorize this - audience = self._app.config['SERVER_HOSTNAME'] + audience = self._server_hostname context, subject = build_context_and_subject() access = [{ 'type': 'repository', @@ -195,10 +196,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): TOKEN_VALIDITY_LIFETIME_S, self._instance_keys) auth_header = 'Bearer ' + auth_token - with self._app.test_request_context('/'): - relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace, - digest=image.storage.content_checksum) - uri = urljoin(get_app_url(self._config), relative_layer_url) + uri = self._uri_creator(repository_and_namespace, image.storage.content_checksum) return uri, auth_header diff --git a/util/secscan/secscan_util.py b/util/secscan/secscan_util.py new file mode 100644 index 000000000..468e37ca1 --- /dev/null +++ b/util/secscan/secscan_util.py @@ -0,0 +1,22 @@ +from urlparse import urljoin + +from flask import url_for + + +def get_blob_download_uri_getter(context, url_scheme_and_hostname): + """ + Returns a function with context to later generate the uri for a download blob + :param context: Flask RequestContext + :param url_scheme_and_hostname: URLSchemeAndHostname class instance + :return: function (repository_and_namespace, checksum) -> uri + """ + def create_uri(repository_and_namespace, checksum): + """ + Creates a uri for a download blob from a repository, namespace, and checksum from earlier context + """ + with context: + relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace, + digest=checksum) + return urljoin(url_scheme_and_hostname.get_url(), relative_layer_url) + + return create_uri diff --git a/util/secscan/test/test_secscan_util.py b/util/secscan/test/test_secscan_util.py new file mode 100644 index 000000000..1e7c222cf --- /dev/null +++ b/util/secscan/test/test_secscan_util.py @@ -0,0 +1,19 @@ +import pytest + +from app import app +from util.config import URLSchemeAndHostname +from util.secscan.secscan_util import get_blob_download_uri_getter + +from test.fixtures import * + +@pytest.mark.parametrize('url_scheme_and_hostname, repo_namespace, checksum, expected_value,', [ + (URLSchemeAndHostname('http', 'localhost:5000'), + 'devtable/simple', 'tarsum+sha256:123', + 'http://localhost:5000/v2/devtable/simple/blobs/tarsum+sha256:123'), +]) +def test_blob_download_uri_getter(app, url_scheme_and_hostname, + repo_namespace, checksum, + expected_value): + blob_uri_getter = get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname) + + assert blob_uri_getter(repo_namespace, checksum) == expected_value diff --git a/util/secscan/validator.py b/util/secscan/validator.py index d8eb61c01..1eaa1244b 100644 --- a/util/secscan/validator.py +++ b/util/secscan/validator.py @@ -1,28 +1,27 @@ import logging -import features - logger = logging.getLogger(__name__) class SecurityConfigValidator(object): """ Helper class for validating the security scanner configuration. """ - def __init__(self, config): - if not features.SECURITY_SCANNER: + def __init__(self, feature_sec_scan, sec_scan_endpoint): + if not feature_sec_scan: return - self._config = config + self._feature_sec_scan = feature_sec_scan + self._sec_scan_endpoint = sec_scan_endpoint def valid(self): - if not features.SECURITY_SCANNER: + if not self._feature_sec_scan: return False - if self._config.get('SECURITY_SCANNER_ENDPOINT') is None: + if self._sec_scan_endpoint is None: logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration') return False - endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT') + endpoint = self._sec_scan_endpoint if not endpoint.startswith('http://') and not endpoint.startswith('https://'): logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https') return False diff --git a/workers/securityworker/securityworker.py b/workers/securityworker/securityworker.py index 552d7fd2f..100308acf 100644 --- a/workers/securityworker/securityworker.py +++ b/workers/securityworker/securityworker.py @@ -19,7 +19,7 @@ DEFAULT_INDEXING_INTERVAL = 30 class SecurityWorker(Worker): def __init__(self): super(SecurityWorker, self).__init__() - validator = SecurityConfigValidator(app.config) + validator = SecurityConfigValidator(app.config.get('FEATURE_SECURITY_SCANNER', False), app.config.get('SECURITY_SCANNER_ENDPOINT')) if not validator.valid(): logger.warning('Failed to validate security scan configuration') return