Add a security scanner api config object for params

Change SecScanAPI to use a uri creation func instead of test context

Pass config provider through validator context

Remove app config dependency for validators

app.py

@@ -35,8 +35,9 @@ from oauth.services.github import GithubOAuthService
 from oauth.services.gitlab import GitLabOAuthService
 from oauth.loginmanager import OAuthLoginManager
 from storage import Storage
+from util.config import URLSchemeAndHostname
 from util.log import filter_logs
-from util import get_app_url
+from util import get_app_url, create_uri_func_from_context
 from util.ipresolver import IPResolver
 from util.saas.analytics import Analytics
 from util.saas.useranalytics import UserAnalytics
@@ -50,7 +51,7 @@ from util.metrics.prometheus import PrometheusPlugin
 from util.saas.cloudwatch import start_cloudwatch_sender
 from util.secscan.api import SecurityScannerAPI
 from util.tufmetadata.api import TUFMetadataAPI
-from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
+from util.security.instancekeys import InstanceKeys
 from util.security.signing import Signer
 
 
@@ -182,7 +183,7 @@ mail = Mail(app)
 prometheus = PrometheusPlugin(app)
 metric_queue = MetricQueue(prometheus)
 chunk_cleanup_queue = WorkQueue(app.config['CHUNK_CLEANUP_QUEUE_NAME'], tf, metric_queue=metric_queue)
-instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config))
+instance_keys = InstanceKeys(app)
 ip_resolver = IPResolver(app)
 storage = Storage(app, metric_queue, chunk_cleanup_queue, instance_keys, config_provider, ip_resolver)
 userfiles = Userfiles(app, storage)
@@ -196,7 +197,7 @@ authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECT
 userevents = UserEventsBuilderModule(app)
 superusers = SuperUserManager(app)
 signer = Signer(app, config_provider)
-instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config))
+instance_keys = InstanceKeys(app)
 label_validator = LabelValidator(app)
 build_canceller = BuildCanceller(app)
 
@@ -228,7 +229,11 @@ namespace_gc_queue = WorkQueue(app.config['NAMESPACE_GC_QUEUE_NAME'], tf, has_na
 all_queues = [image_replication_queue, dockerfile_build_queue, notification_queue,
               secscan_notification_queue, chunk_cleanup_queue, namespace_gc_queue]
 
-secscan_api = SecurityScannerAPI(app, app.config, storage)
+url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
+secscan_api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
+                                 uri_creator=create_uri_func_from_context(app.test_request_context('/'), url_scheme_and_hostname),
+                                 instance_keys=instance_keys)
+
 tuf_metadata_api = TUFMetadataAPI(app, app.config)
 
 # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.

endpoints/api/suconfig.py

@@ -405,7 +405,10 @@ class SuperUserConfigValidate(ApiResource):
     # this is also safe since this method does not access any information not given in the request.
     if not config_provider.config_exists() or SuperUserPermission().can():
       config = request.get_json()['config']
-      validator_context = ValidatorContext.from_app(config, request.get_json().get('password', ''), app, ip_resolver)
+      validator_context = ValidatorContext.from_app(config, request.get_json().get('password', ''), app,
+                                                    ip_resolver=ip_resolver,
+                                                    config_provider=config_provider)
+
       return validate_service_for_config(service, validator_context)
 
     abort(403)

endpoints/githubtrigger.py

@@ -10,7 +10,6 @@ from auth.decorators import require_session_login
 from auth.permissions import AdministerRepositoryPermission
 from data import model
 from endpoints.decorators import route_show_if, parse_repository_name
-from util.config import URLSchemeAndHostname
 from util.http import abort
 
 
@@ -27,7 +26,6 @@ def attach_github_build_trigger(namespace_name, repo_name):
   permission = AdministerRepositoryPermission(namespace_name, repo_name)
   if permission.can():
     code = request.args.get('code')
-    # url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
     token = github_trigger.exchange_code_for_token(app.config, client, code)
     repo = model.repository.get_repository(namespace_name, repo_name)
     if not repo:

endpoints/gitlabtrigger.py

@@ -10,7 +10,6 @@ from auth.decorators import require_session_login
 from auth.permissions import AdministerRepositoryPermission
 from data import model
 from endpoints.decorators import route_show_if
-from util.config import URLSchemeAndHostname
 from util.http import abort
 
 
@@ -35,7 +34,6 @@ def attach_gitlab_build_trigger():
   permission = AdministerRepositoryPermission(namespace, repository)
   if permission.can():
     code = request.args.get('code')
-    # url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
     token = gitlab_trigger.exchange_code_for_token(app.config, client, code,
                                                    redirect_suffix='/trigger')
     if not token:

oauth/login.py

@@ -6,7 +6,6 @@ from six import add_metaclass
 import features
 
 from oauth.base import OAuthService, OAuthExchangeCodeException, OAuthGetUserInfoException
-from util.config import URLSchemeAndHostname
 
 logger = logging.getLogger(__name__)
 
@@ -65,7 +64,6 @@ class OAuthLoginService(OAuthService):
 
     # Retrieve the token for the OAuth code.
     try:
-      # url_scheme_and_hostname = URLSchemeAndHostname(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME'])
       token = self.exchange_code_for_token(app_config, http_client, code,
                                            redirect_suffix=redirect_suffix,
                                            form_encode=self.requires_form_encoding())

oauth/services/github.py

@@ -75,7 +75,8 @@ class GithubOAuthService(OAuthLoginService):
   def orgs_endpoint(self):
     return slash_join(self._api_endpoint(), 'user/orgs')
 
-  def validate_client_id_and_secret(self, http_client, app_config):
+  # TODO(sam): refactor the base method to not take app config
+  def validate_client_id_and_secret(self, http_client):
     # First: Verify that the github endpoint is actually Github by checking for the
     # X-GitHub-Request-Id here.
     api_endpoint = self._api_endpoint()

oauth/services/google.py

@@ -41,7 +41,8 @@ class GoogleOAuthService(OAuthLoginService):
   def requires_form_encoding(self):
     return True
 
-  def validate_client_id_and_secret(self, http_client, app_config):
+  # TODO(sam): this signature does not match its parent class. refactor the base method to take the namedtuple URLSchemeAndHostname
+  def validate_client_id_and_secret(self, http_client):
     # To verify the Google client ID and secret, we hit the
     # https://www.googleapis.com/oauth2/v3/token endpoint with an invalid request. If the client
     # ID or secret are invalid, we get returned a 403 Unauthorized. Otherwise, we get returned

test/test_secscan.py

@@ -8,11 +8,14 @@ from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
 from endpoints.v2 import v2_bp
 from initdb import setup_database_for_testing, finished_database_for_testing
 from notifications.notificationevent import VulnerabilityFoundEvent
+from util import create_uri_func_from_context
+from util.config import URLSchemeAndHostname
 from util.morecollections import AttrDict
 from util.secscan.api import SecurityScannerAPI, APIRequestFailure
 from util.secscan.analyzer import LayerAnalyzer
 from util.secscan.fake import fake_security_scanner
 from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult
+from util.security.instancekeys import InstanceKeys
 from workers.security_notification_worker import SecurityNotificationWorker
 
 
@@ -42,7 +45,13 @@ class TestSecurityScanner(unittest.TestCase):
     self.ctx = app.test_request_context()
     self.ctx.__enter__()
 
-    self.api = SecurityScannerAPI(app, app.config, storage)
+
+    url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
+    instance_keys = InstanceKeys(app)
+    self.api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
+                                  uri_creator=create_uri_func_from_context(app.test_request_context('/'),
+                                                                           url_scheme_and_hostname),
+                                  instance_keys=instance_keys)
 
   def tearDown(self):
     storage.remove(['local_us'], 'supports_direct_download')

test/test_storageproxy.py

@@ -7,7 +7,7 @@ from flask_testing import LiveServerTestCase
 
 from initdb import setup_database_for_testing, finished_database_for_testing
 from storage import Storage
-from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
+from util.security.instancekeys import InstanceKeys
 
 _PORT_NUMBER = 5001
 
@@ -42,7 +42,7 @@ class TestStorageProxy(LiveServerTestCase):
       'test': ['FakeStorage', {}],
     }
 
-    instance_keys = InstanceKeys(instance_keys_context_from_app_config(self.test_app.config))
+    instance_keys = InstanceKeys(self.test_app)
     self.storage = Storage(self.test_app, instance_keys=instance_keys)
     self.test_app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = ['test']
     return self.test_app

util/__init__.py

@@ -1,3 +1,6 @@
+from flask import url_for
+from urlparse import urljoin
+
 def get_app_url(config):
   """ Returns the application's URL, based on the given config. """
   return '%s://%s' % (config['PREFERRED_URL_SCHEME'], config['SERVER_HOSTNAME'])
@@ -19,3 +22,13 @@ def slash_join(*args):
 
   args = [rmslash(path) for path in args]
   return '/'.join(args)
+
+def create_uri_func_from_context(context, url_scheme_and_hostname):
+  def create_uri(repository_and_namespace, checksum):
+    with context:
+      relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
+                                   digest=checksum)
+    return urljoin(get_app_url_from_scheme_hostname(url_scheme_and_hostname), relative_layer_url)
+
+  return create_uri
+

util/config/provider/basefileprovider.py

@@ -68,3 +68,7 @@ class BaseFileProvider(BaseProvider):
         return True
 
     return False
+
+  def get_config_root(self):
+    return self.config_volume
+

util/config/provider/baseprovider.py

@@ -123,3 +123,8 @@ class BaseProvider(object):
   def get_volume_path(self, directory, filename):
     """ Helper for constructing relative file paths, which may differ between providers.
         For example, kubernetes can't have subfolders in configmaps """
+
+  @abstractmethod
+  def get_config_root(self):
+    """ Returns the config root directory. """
+

util/config/provider/testprovider.py

@@ -10,6 +10,11 @@ REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', '
 class TestConfigProvider(BaseProvider):
   """ Implementation of the config provider for testing. Everything is kept in-memory instead on
       the real file system. """
+
+  def get_config_root(self):
+    raise Exception('Test Config does not have a config root')
+    # return ''
+
   def __init__(self):
     self.clear()
 

util/config/validator.py

@@ -2,6 +2,7 @@ import logging
 
 from auth.auth_context import get_authenticated_user
 from data.users import LDAP_CERT_FILENAME
+from util import create_uri_func_from_context
 from util.config import URLSchemeAndHostname
 
 from util.config.validators.validate_database import DatabaseValidator
@@ -99,22 +100,40 @@ class ValidatorContext(object):
   """ Context to run validators in, with any additional runtime configuration they need
   """
   def __init__(self, config, user_password=None, http_client=None, context=None,
-               url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, ip_resolver=None):
+               url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
+               ip_resolver=None, feature_sec_scanner=False, is_testing=False,
+               uri_creator=None, config_provider=None):
     self.config = config
     self.user = get_authenticated_user()
     self.user_password = user_password
     self.http_client = http_client
     self.context = context
-    self.scheme_and_hostname = url_scheme_and_hostname
+    self.url_scheme_and_hostname = url_scheme_and_hostname
     self.jwt_auth_max = jwt_auth_max
     self.registry_title = registry_title
     self.ip_resolver = ip_resolver
+    self.feature_sec_scanner = feature_sec_scanner
+    self.is_testing = is_testing
+    self.uri_creator = uri_creator
+    self.config_provider = config_provider
 
   @classmethod
-  def from_app(cls, config, user_password, app, ip_resolver):
+  def from_app(cls, config, user_password, app, ip_resolver, client=None, config_provider=None):
-    cls(config, user_password, app.config['HTTP_CLIENT'], app.app_context,
+    url_scheme = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
-        URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']),
+
-        app.config.get('JWT_AUTH_MAX_FRESH_S', 300), app.config['REGISTRY_TITLE'], ip_resolver)
+    cls(config,
+        user_password,
+        client or app.config['HTTPCLIENT'],
+        app.app_context,
+        url_scheme,
+        app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
+        app.config['REGISTRY_TITLE'],
+        ip_resolver,
+        app.config.get('FEATURE_SECURITY_SCANNER', False),
+        app.config.get('TESTING', False),
+        create_uri_func_from_context(app.test_request_context('/'), url_scheme),
+        config_provider)
+
 
 
 

util/config/validators/test/test_validate_access.py

@@ -1,5 +1,6 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_access import AccessSettingsValidator
 
@@ -17,6 +18,6 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, expected_excepti
 
   if expected_exception is not None:
     with pytest.raises(expected_exception):
-      validator.validate(unvalidated_config, None, None)
+      validator.validate(ValidatorContext(unvalidated_config))
   else:
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(ValidatorContext(unvalidated_config))

util/config/validators/test/test_validate_actionlog_archiving.py

@@ -1,5 +1,6 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
 
@@ -12,7 +13,7 @@ from test.fixtures import *
 ])
 def test_skip_validate_actionlog(unvalidated_config, app):
   validator = ActionLogArchivingValidator()
-  validator.validate(unvalidated_config, None, None)
+  validator.validate(ValidatorContext(unvalidated_config))
 
 
 @pytest.mark.parametrize('config, expected_error', [
@@ -33,19 +34,19 @@ def test_invalid_config(config, expected_error, app):
   validator = ActionLogArchivingValidator()
 
   with pytest.raises(ConfigValidationException) as ipe:
-    validator.validate(config, None, None)
+    validator.validate(ValidatorContext(config))
 
   assert ipe.value.message == expected_error
 
 def test_valid_config(app):
-  config = {
+  config = ValidatorContext({
     'FEATURE_ACTION_LOG_ROTATION': True,
     'ACTION_LOG_ARCHIVE_PATH': 'somepath',
     'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
     'DISTRIBUTED_STORAGE_CONFIG': {
       'somelocation': {},
     },
-  }
+  })
 
   validator = ActionLogArchivingValidator()
-  validator.validate(config, None, None)
+  validator.validate(config)

util/config/validators/test/test_validate_apptokenauth.py

@@ -1,5 +1,6 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
 
@@ -15,15 +16,15 @@ def test_validate_invalid_auth_config(unvalidated_config, app):
   validator = AppTokenAuthValidator()
 
   with pytest.raises(ConfigValidationException):
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(ValidatorContext(unvalidated_config))
 
 
 def test_validate_auth(app):
-  config = {
+  config = ValidatorContext({
     'AUTHENTICATION_TYPE': 'AppToken',
     'FEATURE_APP_SPECIFIC_TOKENS': True,
     'FEATURE_DIRECT_LOGIN': False,
-  }
+  })
 
   validator = AppTokenAuthValidator()
-  validator.validate(config, None, None)
+  validator.validate(config)

util/config/validators/test/test_validate_bitbucket_trigger.py

@@ -2,22 +2,24 @@ import pytest
 
 from httmock import urlmatch, HTTMock
 
+from util.config import URLSchemeAndHostname
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
 
 from test.fixtures import *
 
 @pytest.mark.parametrize('unvalidated_config', [
-  ({}),
+  (ValidatorContext({})),
-  ({'BITBUCKET_TRIGGER_CONFIG': {}}),
+  (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
-  ({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}}),
+  (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
-  ({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}}),
+  (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
 ])
 def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
   validator = BitbucketTriggerValidator()
 
   with pytest.raises(ConfigValidationException):
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(unvalidated_config)
 
 def test_validate_bitbucket_trigger(app):
   url_hit = [False]
@@ -32,11 +34,16 @@ def test_validate_bitbucket_trigger(app):
 
   with HTTMock(handler):
     validator = BitbucketTriggerValidator()
-    validator.validate({
+
+    unvalidated_config = ValidatorContext({
       'BITBUCKET_TRIGGER_CONFIG': {
         'CONSUMER_KEY': 'foo',
         'CONSUMER_SECRET': 'bar',
       },
-    }, None, None)
+    })
+
+    unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
+
+    validator.validate(unvalidated_config)
 
     assert url_hit[0]

util/config/validators/test/test_validate_database.py

@@ -1,22 +1,23 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_database import DatabaseValidator
 
 from test.fixtures import *
 
 @pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
-  (None, None, None, TypeError),
+  (ValidatorContext(None), None, None, TypeError),
-  ({}, None, None, KeyError),
+  (ValidatorContext({}), None, None, KeyError),
-  ({'DB_URI': 'sqlite:///:memory:'}, None, None, None),
+  (ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
-  ({'DB_URI': 'invalid:///:memory:'}, None, None, KeyError),
+  (ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
-  ({'DB_NOTURI': 'sqlite:///:memory:'}, None, None, KeyError),
+  (ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
 ])
 def test_validate_database(unvalidated_config, user, user_password, expected, app):
   validator = DatabaseValidator()
 
   if expected is not None:
     with pytest.raises(expected):
-      validator.validate(unvalidated_config, user, user_password)
+      validator.validate(unvalidated_config)
   else:
-    validator.validate(unvalidated_config, user, user_password)
+    validator.validate(unvalidated_config)

util/config/validators/test/test_validate_github.py

@@ -2,6 +2,8 @@ import pytest
 
 from httmock import urlmatch, HTTMock
 
+from config import build_requests_session
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
 
@@ -36,7 +38,7 @@ def test_validate_invalid_github_config(github_config, github_validator, app):
   with pytest.raises(ConfigValidationException):
     unvalidated_config = {}
     unvalidated_config[github_validator.config_key] = github_config
-    github_validator.validate(unvalidated_config, None, None)
+    github_validator.validate(ValidatorContext(unvalidated_config))
 
 def test_validate_github(github_validator, app):
   url_hit = [False, False]
@@ -52,13 +54,16 @@ def test_validate_github(github_validator, app):
     return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
 
   with HTTMock(app_handler, handler):
-    github_validator.validate({
+    unvalidated_config = ValidatorContext({
       github_validator.config_key: {
         'GITHUB_ENDPOINT': 'http://somehost',
         'CLIENT_ID': 'foo',
         'CLIENT_SECRET': 'bar',
       },
-    }, None, None)
+    })
+
+    unvalidated_config.http_client = build_requests_session()
+    github_validator.validate(unvalidated_config)
 
   assert url_hit[0]
   assert url_hit[1]

util/config/validators/test/test_validate_gitlab_trigger.py

@@ -3,6 +3,9 @@ import pytest
 
 from httmock import urlmatch, HTTMock
 
+from config import build_requests_session
+from util.config import URLSchemeAndHostname
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
 
@@ -18,7 +21,7 @@ def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
   validator = GitLabTriggerValidator()
 
   with pytest.raises(ConfigValidationException):
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(ValidatorContext(unvalidated_config))
 
 def test_validate_gitlab_enterprise_trigger(app):
   url_hit = [False]
@@ -30,12 +33,16 @@ def test_validate_gitlab_enterprise_trigger(app):
 
   with HTTMock(handler):
     validator = GitLabTriggerValidator()
-    validator.validate({
+    unvalidated_config = ValidatorContext({
       'GITLAB_TRIGGER_CONFIG': {
         'GITLAB_ENDPOINT': 'http://somegitlab',
         'CLIENT_ID': 'foo',
         'CLIENT_SECRET': 'bar',
       },
-    }, None, None)
+    })
+    unvalidated_config.http_client = build_requests_session()
+
+    unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
+    validator.validate(unvalidated_config)
 
   assert url_hit[0]

util/config/validators/test/test_validate_google_login.py

@@ -2,6 +2,8 @@ import pytest
 
 from httmock import urlmatch, HTTMock
 
+from config import build_requests_session
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_google_login import GoogleLoginValidator
 
@@ -17,7 +19,7 @@ def test_validate_invalid_google_login_config(unvalidated_config, app):
   validator = GoogleLoginValidator()
 
   with pytest.raises(ConfigValidationException):
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(ValidatorContext(unvalidated_config))
 
 def test_validate_google_login(app):
   url_hit = [False]
@@ -29,11 +31,15 @@ def test_validate_google_login(app):
   validator = GoogleLoginValidator()
 
   with HTTMock(handler):
-    validator.validate({
+    unvalidated_config = ValidatorContext({
       'GOOGLE_LOGIN_CONFIG': {
         'CLIENT_ID': 'foo',
         'CLIENT_SECRET': 'bar',
       },
-    }, None, None)
+    })
+
+    unvalidated_config.http_client = build_requests_session()
+
+    validator.validate(unvalidated_config)
 
   assert url_hit[0]

util/config/validators/test/test_validate_jwt.py

@@ -1,5 +1,7 @@
 import pytest
 
+from config import build_requests_session
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_jwt import JWTAuthValidator
 from util.morecollections import AttrDict
@@ -7,6 +9,7 @@ from util.morecollections import AttrDict
 from test.test_external_jwt_authn import fake_jwt
 
 from test.fixtures import *
+from app import config_provider
 
 
 @pytest.mark.parametrize('unvalidated_config', [
@@ -14,7 +17,9 @@ from test.fixtures import *
   ({'AUTHENTICATION_TYPE': 'Database'}),
 ])
 def test_validate_noop(unvalidated_config, app):
-  JWTAuthValidator.validate(unvalidated_config, None, None, app)
+  config = ValidatorContext(unvalidated_config)
+  config.config_provider = config_provider
+  JWTAuthValidator.validate(config)
 
 
 @pytest.mark.parametrize('unvalidated_config', [
@@ -24,7 +29,9 @@ def test_validate_noop(unvalidated_config, app):
 ])
 def test_invalid_config(unvalidated_config, app):
   with pytest.raises(ConfigValidationException):
-    JWTAuthValidator.validate(unvalidated_config, None, None, app)
+    config = ValidatorContext(unvalidated_config)
+    config.config_provider = config_provider
+    JWTAuthValidator.validate(config)
 
 
 @pytest.mark.parametrize('username, password, expected_exception', [
@@ -42,10 +49,15 @@ def test_validated_jwt(username, password, expected_exception, app):
     config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
     config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
 
+    unvalidated_config = ValidatorContext(config)
+    unvalidated_config.user = AttrDict(dict(username=username))
+    unvalidated_config.user_password = password
+    unvalidated_config.config_provider = config_provider
+
+    unvalidated_config.http_client = build_requests_session()
+
     if expected_exception is not None:
       with pytest.raises(ConfigValidationException):
-        JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app,
+        JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
-                                  public_key_path=jwt_auth.public_key_path)
     else:
-        JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app,
+        JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
-                                  public_key_path=jwt_auth.public_key_path)

util/config/validators/test/test_validate_keystone.py

@@ -1,5 +1,6 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_keystone import KeystoneValidator
 from util.morecollections import AttrDict
@@ -13,7 +14,7 @@ from test.fixtures import *
   ({'AUTHENTICATION_TYPE': 'Database'}),
 ])
 def test_validate_noop(unvalidated_config, app):
-  KeystoneValidator.validate(unvalidated_config, None, None)
+  KeystoneValidator.validate(ValidatorContext(unvalidated_config))
 
 @pytest.mark.parametrize('unvalidated_config', [
   ({'AUTHENTICATION_TYPE': 'Keystone'}),
@@ -25,7 +26,7 @@ def test_validate_noop(unvalidated_config, app):
 ])
 def test_invalid_config(unvalidated_config, app):
   with pytest.raises(ConfigValidationException):
-    KeystoneValidator.validate(unvalidated_config, None, None)
+    KeystoneValidator.validate(ValidatorContext(unvalidated_config))
 
 
 @pytest.mark.parametrize('username, password, expected_exception', [
@@ -45,8 +46,12 @@ def test_validated_keystone(username, password, expected_exception, app):
     config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
     config['KEYSTONE_ADMIN_TENANT'] = 'admintenant'
 
+    unvalidated_config = ValidatorContext(config)
+    unvalidated_config.user = AttrDict(dict(username=username))
+    unvalidated_config.user_password = password
+
     if expected_exception is not None:
       with pytest.raises(ConfigValidationException):
-        KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
+        KeystoneValidator.validate(unvalidated_config)
     else:
-      KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
+      KeystoneValidator.validate(unvalidated_config)

util/config/validators/test/test_validate_ldap.py

@@ -1,5 +1,6 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_ldap import LDAPValidator
 from util.morecollections import AttrDict
@@ -7,13 +8,16 @@ from util.morecollections import AttrDict
 from test.test_ldap import mock_ldap
 
 from test.fixtures import *
+from app import config_provider
 
 @pytest.mark.parametrize('unvalidated_config', [
   ({}),
   ({'AUTHENTICATION_TYPE': 'Database'}),
 ])
 def test_validate_noop(unvalidated_config, app):
-  LDAPValidator.validate(unvalidated_config, None, None)
+  config = ValidatorContext(unvalidated_config)
+  config.config_provider = config_provider
+  LDAPValidator.validate(config)
 
 @pytest.mark.parametrize('unvalidated_config', [
   ({'AUTHENTICATION_TYPE': 'LDAP'}),
@@ -21,7 +25,9 @@ def test_validate_noop(unvalidated_config, app):
 ])
 def test_invalid_config(unvalidated_config, app):
   with pytest.raises(ConfigValidationException):
-    LDAPValidator.validate(unvalidated_config, None, None)
+    config = ValidatorContext(unvalidated_config)
+    config.config_provider = config_provider
+    LDAPValidator.validate(config)
 
 
 @pytest.mark.parametrize('uri', [
@@ -39,7 +45,9 @@ def test_invalid_uri(uri, app):
   config['LDAP_URI'] = uri
 
   with pytest.raises(ConfigValidationException):
-    LDAPValidator.validate(config, None, None)
+    config = ValidatorContext(config)
+    config.config_provider = config_provider
+    LDAPValidator.validate(config)
 
 
 @pytest.mark.parametrize('username, password, expected_exception', [
@@ -56,10 +64,15 @@ def test_validated_ldap(username, password, expected_exception, app):
   config['LDAP_ADMIN_PASSWD'] = 'password'
   config['LDAP_USER_RDN'] = ['ou=employees']
 
+  unvalidated_config = ValidatorContext(config)
+  unvalidated_config.user = AttrDict(dict(username=username))
+  unvalidated_config.user_password = password
+  unvalidated_config.config_provider = config_provider
+
   if expected_exception is not None:
     with pytest.raises(ConfigValidationException):
       with mock_ldap():
-        LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
+        LDAPValidator.validate(unvalidated_config)
   else:
     with mock_ldap():
-      LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
+      LDAPValidator.validate(unvalidated_config)

util/config/validators/test/test_validate_oidc.py

@@ -3,7 +3,9 @@ import pytest
 
 from httmock import urlmatch, HTTMock
 
+from config import build_requests_session
 from oauth.oidc import OIDC_WELLKNOWN
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_oidc import OIDCLoginValidator
 
@@ -19,7 +21,7 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, app):
   validator = OIDCLoginValidator()
 
   with pytest.raises(ConfigValidationException):
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(ValidatorContext(unvalidated_config))
 
 def test_validate_oidc_login(app):
   url_hit = [False]
@@ -33,13 +35,16 @@ def test_validate_oidc_login(app):
 
   with HTTMock(handler):
     validator = OIDCLoginValidator()
-    validator.validate({
+    unvalidated_config = ValidatorContext({
       'SOMETHING_LOGIN_CONFIG': {
         'CLIENT_ID': 'foo',
         'CLIENT_SECRET': 'bar',
         'OIDC_SERVER': 'http://someserver',
         'DEBUGGING': True, # Allows for HTTP.
       },
-    }, None, None)
+    })
+    unvalidated_config.http_client = build_requests_session()
+
+    validator.validate(unvalidated_config)
 
   assert url_hit[0]

util/config/validators/test/test_validate_redis.py

@@ -5,10 +5,13 @@ from mock import patch
 
 from mockredis import mock_strict_redis_client
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_redis import RedisValidator
 
 from test.fixtures import *
+from util.morecollections import AttrDict
+
 
 @pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
   ({}, None, None, False, ConfigValidationException),
@@ -19,8 +22,13 @@ from test.fixtures import *
 def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
   with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
     validator = RedisValidator()
+    unvalidated_config = ValidatorContext(unvalidated_config)
+
+    unvalidated_config.user = AttrDict(dict(username=user))
+    unvalidated_config.user_password = user_password
+
     if expected is not None:
       with pytest.raises(expected):
-        validator.validate(unvalidated_config, user, user_password)
+        validator.validate(unvalidated_config)
     else:
-      validator.validate(unvalidated_config, user, user_password)
+      validator.validate(unvalidated_config)

util/config/validators/test/test_validate_secscan.py

@@ -1,6 +1,8 @@
 import pytest
 
-from util.config.validators import ConfigValidationException
+from config import build_requests_session
+from util.config import URLSchemeAndHostname
+from util.config.validator import ValidatorContext
 from util.config.validators.validate_secscan import SecurityScannerValidator
 from util.secscan.fake import fake_security_scanner
 
@@ -10,7 +12,11 @@ from test.fixtures import *
   ({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
 ])
 def test_validate_noop(unvalidated_config, app):
-  SecurityScannerValidator.validate(unvalidated_config, None, None)
+  unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True)
+  unvalidated_config.http_client = build_requests_session()
+  unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
+
+  SecurityScannerValidator.validate(unvalidated_config)
 
 
 @pytest.mark.parametrize('unvalidated_config, expected_error', [
@@ -29,9 +35,13 @@ def test_validate_noop(unvalidated_config, app):
   }, None),
 ])
 def test_validate(unvalidated_config, expected_error, app):
+  unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True)
+  unvalidated_config.http_client = build_requests_session()
+  unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
+
   with fake_security_scanner(hostname='fakesecurityscanner'):
     if expected_error is not None:
       with pytest.raises(expected_error):
-        SecurityScannerValidator.validate(unvalidated_config, None, None)
+        SecurityScannerValidator.validate(unvalidated_config)
     else:
-      SecurityScannerValidator.validate(unvalidated_config, None, None)
+      SecurityScannerValidator.validate(unvalidated_config)

util/config/validators/test/test_validate_signer.py

@@ -1,5 +1,6 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_signer import SignerValidator
 
@@ -14,6 +15,6 @@ def test_validate_signer(unvalidated_config, expected, app):
   validator = SignerValidator()
   if expected is not None:
     with pytest.raises(expected):
-      validator.validate(unvalidated_config, None, None)
+      validator.validate(ValidatorContext(unvalidated_config))
   else:
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(ValidatorContext(unvalidated_config))

util/config/validators/test/test_validate_ssl.py

@@ -3,11 +3,13 @@ import pytest
 from mock import patch
 from tempfile import NamedTemporaryFile
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
 from test.test_ssl_util import generate_test_cert
 
 from test.fixtures import *
+from app import config_provider
 
 @pytest.mark.parametrize('unvalidated_config', [
   ({}),
@@ -16,7 +18,7 @@ from test.fixtures import *
 ])
 def test_skip_validate_ssl(unvalidated_config, app):
   validator = SSLValidator()
-  validator.validate(unvalidated_config, None, None)
+  validator.validate(ValidatorContext(unvalidated_config))
 
 
 @pytest.mark.parametrize('cert, expected_error, error_message', [
@@ -54,11 +56,13 @@ def test_validate_ssl(cert, expected_error, error_message, app):
     with patch('app.config_provider.volume_file_exists', return_true):
       with patch('app.config_provider.get_volume_file', get_volume_file):
         validator = SSLValidator()
+        config = ValidatorContext(config)
+        config.config_provider = config_provider
 
         if expected_error is not None:
           with pytest.raises(expected_error) as ipe:
-            validator.validate(config, None, None)
+            validator.validate(config)
 
           assert ipe.value.message == error_message
         else:
-          validator.validate(config, None, None)
+          validator.validate(config)

util/config/validators/test/test_validate_storage.py

@@ -1,6 +1,7 @@
 import moto
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_storage import StorageValidator
 
@@ -16,15 +17,15 @@ def test_validate_storage(unvalidated_config, expected, app):
   validator = StorageValidator()
   if expected is not None:
     with pytest.raises(expected):
-      validator.validate(unvalidated_config, None, None)
+      validator.validate(ValidatorContext(unvalidated_config))
   else:
-    validator.validate(unvalidated_config, None, None)
+    validator.validate(ValidatorContext(unvalidated_config))
 
 def test_validate_s3_storage(app):
   validator = StorageValidator()
   with moto.mock_s3():
     with pytest.raises(ConfigValidationException) as ipe:
-      validator.validate({
+      validator.validate(ValidatorContext({
         'DISTRIBUTED_STORAGE_CONFIG': {
           'default': ('S3Storage', {
             's3_access_key': 'invalid',
@@ -33,6 +34,6 @@ def test_validate_s3_storage(app):
             'storage_path': ''
           }),
         }
-      }, None, None)
+      }))
 
     assert ipe.value.message == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'

util/config/validators/test/test_validate_timemachine.py

@@ -1,5 +1,6 @@
 import pytest
 
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_timemachine import TimeMachineValidator
 
@@ -7,7 +8,7 @@ from util.config.validators.validate_timemachine import TimeMachineValidator
   ({}),
 ])
 def test_validate_noop(unvalidated_config):
-  TimeMachineValidator.validate(unvalidated_config, None, None)
+  TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
 
 
 from test.fixtures import *
@@ -25,7 +26,7 @@ def test_validate(default_exp, options, expected_exception, app):
 
   if expected_exception is not None:
     with pytest.raises(ConfigValidationException) as cve:
-      TimeMachineValidator.validate(config, None, None)
+      TimeMachineValidator.validate(ValidatorContext(config))
     assert str(cve.value) == str(expected_exception)
   else:
-    TimeMachineValidator.validate(config, None, None)
+    TimeMachineValidator.validate(ValidatorContext(config))

util/config/validators/test/test_validate_torrent.py

@@ -2,6 +2,8 @@ import pytest
 
 from httmock import urlmatch, HTTMock
 
+from config import build_requests_session
+from util.config.validator import ValidatorContext
 from util.config.validators import ConfigValidationException
 from util.config.validators.validate_torrent import BittorrentValidator
 
@@ -23,8 +25,14 @@ def test_validate_torrent(unvalidated_config, expected, app):
     validator = BittorrentValidator()
     if expected is not None:
       with pytest.raises(expected):
-        validator.validate(unvalidated_config, None, None)
+        config = ValidatorContext(unvalidated_config)
+        config.http_client = build_requests_session()
+
+        validator.validate(config)
       assert not announcer_hit[0]
     else:
-      validator.validate(unvalidated_config, None, None)
+      config = ValidatorContext(unvalidated_config)
+      config.http_client = build_requests_session()
+
+      validator.validate(config)
       assert announcer_hit[0]

util/config/validators/validate_bitbucket_trigger.py

@@ -23,7 +23,7 @@ class BitbucketTriggerValidator(BaseValidator):
 
     key = trigger_config['CONSUMER_KEY']
     secret = trigger_config['CONSUMER_SECRET']
-    callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url(validator_context.scheme_and_hostname))
+    callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url_from_scheme_hostname(validator_context.url_scheme_and_hostname))
 
     bitbucket_client = BitBucket(key, secret, callback_url)
     (result, _, _) = bitbucket_client.get_authorization_url()

util/config/validators/validate_google_login.py

@@ -21,7 +21,6 @@ class GoogleLoginValidator(BaseValidator):
       raise ConfigValidationException('Missing Client Secret')
 
     oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
-    # TODO(sam): the google oauth doesn't need the app config, but when refactoring pass in the URLSchemeandHostname
     result = oauth.validate_client_id_and_secret(client)
     if not result:
       raise ConfigValidationException('Invalid client id or client secret')

util/config/validators/validate_jwt.py

@@ -1,4 +1,4 @@
-from _init import OVERRIDE_CONFIG_DIRECTORY
+import os
 from data.users.externaljwt import ExternalJWTAuthN
 from util.config.validators import BaseValidator, ConfigValidationException
 
@@ -13,6 +13,7 @@ class JWTAuthValidator(BaseValidator):
     user_password = validator_context.user_password
     http_client = validator_context.http_client
     jwt_auth_max = validator_context.jwt_auth_max
+    config_provider = validator_context.config_provider
 
     if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
       return
@@ -29,10 +30,13 @@ class JWTAuthValidator(BaseValidator):
     if not issuer:
       raise ConfigValidationException('Missing JWT Issuer ID')
 
+
+    override_config_directory = os.path.join(config_provider.get_config_root(), 'stack/')
+
     # Try to instatiate the JWT authentication mechanism. This will raise an exception if
     # the key cannot be found.
     users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
-                             OVERRIDE_CONFIG_DIRECTORY,
+                             override_config_directory,
                              http_client,
                              jwt_auth_max,
                              public_key_path=public_key_path,

util/config/validators/validate_ldap.py

@@ -5,7 +5,6 @@ import subprocess
 from data.users import LDAP_CERT_FILENAME
 from data.users.externalldap import LDAPConnection, LDAPUsers
 from util.config.validators import BaseValidator, ConfigValidationException
-from _init import CONF_DIR, config_provider
 
 class LDAPValidator(BaseValidator):
   name = "ldap"
@@ -16,12 +15,14 @@ class LDAPValidator(BaseValidator):
     config = validator_context.config
     user = validator_context.user
     user_password = validator_context.user_password
+    config_provider = validator_context.config_provider
+
     if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
       return
 
     # If there is a custom LDAP certificate, then reinstall the certificates for the container.
     if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
-      subprocess.check_call([os.path.join(CONF_DIR, 'init/certs_install.sh')])
+      subprocess.check_call([os.path.join(config_provider.get_config_root(), '../init/certs_install.sh')])
 
     # Note: raises ldap.INVALID_CREDENTIALS on failure
     admin_dn = config.get('LDAP_ADMIN_DN')

util/config/validators/validate_secscan.py

@@ -12,14 +12,18 @@ class SecurityScannerValidator(BaseValidator):
     """ Validates the configuration for talking to a Quay Security Scanner. """
     config = validator_context.config
     client = validator_context.http_client
-    app = None #TODO(sam) validate with joey's pr about security scanner api
+    feature_sec_scanner = validator_context.feature_sec_scanner
+    is_testing = validator_context.is_testing
 
-    if not config.get('FEATURE_SECURITY_SCANNER', False):
+    server_hostname = validator_context.url_scheme_and_hostname.hostname
+    uri_creator = validator_context.uri_creator
+
+    if not feature_sec_scanner:
       return
 
-    api = SecurityScannerAPI(app.config, config, None, client=client, skip_validation=True)
+    api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
 
-    if not config.get('TESTING', False):
+    if not is_testing:
       # Generate a temporary Quay key to use for signing the outgoing requests.
       setup_jwt_proxy()
 

util/config/validators/validate_signer.py

@@ -1,6 +1,5 @@
 from StringIO import StringIO
 
-from _init import config_provider
 from util.config.validators import BaseValidator, ConfigValidationException
 from util.security.signing import SIGNING_ENGINES
 
@@ -11,6 +10,7 @@ class SignerValidator(BaseValidator):
   def validate(cls, validator_context):
     """ Validates the GPG public+private key pair used for signing converted ACIs. """
     config = validator_context.config
+    config_provider = validator_context.config_provider
 
     if config.get('SIGNING_ENGINE') is None:
       return

util/config/validators/validate_ssl.py

@@ -1,4 +1,3 @@
-from _init import config_provider
 from util.config.validators import BaseValidator, ConfigValidationException
 from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
 
@@ -11,6 +10,7 @@ class SSLValidator(BaseValidator):
   def validate(cls, validator_context):
     """ Validates the SSL configuration (if enabled). """
     config = validator_context.config
+    config_provider = validator_context.config_provider
 
     # Skip if non-SSL.
     if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':

util/config/validators/validate_storage.py

@@ -1,4 +1,3 @@
-from _init import config_provider
 from storage import get_storage_driver
 from util.config.validators import BaseValidator, ConfigValidationException
 
@@ -12,10 +11,12 @@ class StorageValidator(BaseValidator):
     config = validator_context.config
     client = validator_context.http_client
     ip_resolver = validator_context.ip_resolver
+    config_provider = validator_context.config_provider
 
+    # replication_enabled = app.config.get('FEATURE_STORAGE_REPLICATION', False)
     replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
 
-    providers = _get_storage_providers(config, ip_resolver).items()
+    providers = _get_storage_providers(config, ip_resolver, config_provider).items()
     if not providers:
       raise ConfigValidationException('Storage configuration required')
 
@@ -35,7 +36,7 @@ class StorageValidator(BaseValidator):
         raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
 
 
-def _get_storage_providers(config, ip_resolver):
+def _get_storage_providers(config, ip_resolver, config_provider):
   storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
   drivers = {}
 

util/secscan/__init__.py

@@ -107,12 +107,3 @@ def get_priority_for_index(index):
       return priority
 
   return 'Unknown'
-
-
-def create_url_from_app(app):
-  """
-  Higher order function that returns a function that when called, will generate a url for that given app
-  :param app: Flask app
-  :return:
-  type: Flask -> (str -> url)
-  """

util/secscan/api.py

@@ -7,16 +7,12 @@ from urlparse import urljoin
 
 import requests
 
-from flask import url_for
-
 from data.database import CloseForLongOperation
 from data import model
 from data.model.storage import get_storage_locations
-from util import get_app_url, slash_join
 from util.abchelpers import nooper
 from util.failover import failover, FailoverException
 from util.secscan.validator import SecurityConfigValidator
-from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
 from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
 
 from _init import CONF_DIR
@@ -70,16 +66,16 @@ def compute_layer_id(layer):
 
 class SecurityScannerAPI(object):
   """ Helper class for talking to the Security Scan service (usually Clair). """
-  def __init__(self, app, config, storage, client=None, skip_validation=False):
+  def __init__(self, config, storage, server_hostname=None, client=None, skip_validation=False, uri_creator=None, instance_keys=None):
     feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False)
     has_valid_config = skip_validation
 
     if not skip_validation and feature_enabled:
-      config_validator = SecurityConfigValidator(config)
+      config_validator = SecurityConfigValidator(feature_enabled, config.get('SECURITY_SCANNER_ENDPOINT'))
       has_valid_config = config_validator.valid()
 
     if feature_enabled and has_valid_config:
-      self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client)
+      self.state = ImplementedSecurityScannerAPI(config, storage, server_hostname, client=client, uri_creator=uri_creator, instance_keys=instance_keys)
     else:
       self.state = NoopSecurityScannerAPI()
 
@@ -150,20 +146,25 @@ class NoopSecurityScannerAPI(SecurityScannerAPIInterface):
 
 class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
   """ Helper class for talking to the Security Scan service (Clair). """
-  def __init__(self, app_config, config, storage, client=None):
+  # TODO(sam) refactor this to not take an app config, and instead just the things it needs as a config object
-    self._app_config = app_config
+  def __init__(self, config, storage, server_hostname, client=None, uri_creator=None, instance_keys=None):
     self._config = config
-    self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app_config))
+    self._instance_keys = instance_keys
-    self._client = client or config['HTTPCLIENT']
+    self._client = client
     self._storage = storage
+    self._server_hostname = server_hostname
     self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
     self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
+    self._uri_creator = uri_creator
 
   def _get_image_url_and_auth(self, image):
     """ Returns a tuple of the url and the auth header value that must be used
         to fetch the layer data itself. If the image can't be addressed, we return
         None.
     """
+    if self._instance_keys is None:
+      raise Exception('No Instance keys provided to Security Scanner API')
+
     path = model.storage.get_layer_path(image.storage)
     locations = self._default_storage_locations
 
@@ -183,7 +184,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
       repository_and_namespace = '/'.join([namespace_name, repo_name])
 
       # Generate the JWT which will authorize this
-      audience = self._app_config['SERVER_HOSTNAME']
+      audience = self._server_hostname
       context, subject = build_context_and_subject()
       access = [{
         'type': 'repository',
@@ -195,10 +196,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
                                          TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
       auth_header = 'Bearer ' + auth_token
 
-      with self._app.test_request_context('/'):
+      uri = self._uri_creator(repository_and_namespace, image.storage.content_checksum)
-        relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
-                                     digest=image.storage.content_checksum)
-      uri = urljoin(get_app_url(self._config), relative_layer_url)
 
     return uri, auth_header
 

util/secscan/validator.py

@@ -1,28 +1,27 @@
 import logging
 
-import features
-
 
 logger = logging.getLogger(__name__)
 
 
 class SecurityConfigValidator(object):
   """ Helper class for validating the security scanner configuration. """
-  def __init__(self, config):
+  def __init__(self, feature_sec_scan, sec_scan_endpoint):
-    if not features.SECURITY_SCANNER:
+    if not feature_sec_scan:
       return
 
-    self._config = config
+    self._feature_sec_scan = feature_sec_scan
+    self._sec_scan_endpoint = sec_scan_endpoint
 
   def valid(self):
-    if not features.SECURITY_SCANNER:
+    if not self._feature_sec_scan:
       return False
 
-    if self._config.get('SECURITY_SCANNER_ENDPOINT') is None:
+    if self._sec_scan_endpoint is None:
       logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
       return False
 
-    endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT')
+    endpoint = self._sec_scan_endpoint
     if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
       logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
       return False

util/security/instancekeys.py

@@ -1,4 +1,3 @@
-from collections import namedtuple
 from cachetools import lru_cache
 from data import model
 from util.expiresdict import ExpiresDict, ExpiresEntry
@@ -26,10 +25,9 @@ class InstanceKeys(object):
   """ InstanceKeys defines a helper class for interacting with the Quay instance service keys
       used for JWT signing of registry tokens as well as requests from Quay to other services
       such as Clair. Each container will have a single registered instance key.
-      :param keys_context: InstanceKeysContext
   """
-  def __init__(self, keys_context):
+  def __init__(self, app):
-    self.keys_context = keys_context
+    self.app = app
     self.instance_keys = ExpiresDict(self._load_instance_keys)
 
   def clear_cache(self):
@@ -47,24 +45,24 @@ class InstanceKeys(object):
   @property
   def service_name(self):
     """ Returns the name of the instance key's service (i.e. 'quay'). """
-    return self.keys_context.instance_key_service
+    return self.app.config['INSTANCE_SERVICE_KEY_SERVICE']
 
   @property
   def service_key_expiration(self):
     """ Returns the defined expiration for instance service keys, in minutes. """
-    return self.keys_context.service_key_expiration
+    return self.app.config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120)
 
   @property
   @lru_cache(maxsize=1)
   def local_key_id(self):
     """ Returns the ID of the local instance service key. """
-    return _load_file_contents(self.keys_context.service_key_kid_location)
+    return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_KID_LOCATION'])
 
   @property
   @lru_cache(maxsize=1)
   def local_private_key(self):
     """ Returns the private key of the local instance service key. """
-    return _load_file_contents(self.keys_context.service_key_location)
+    return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_LOCATION'])
 
   def get_service_key_public_key(self, kid):
     """ Returns the public key associated with the given instance service key or None if none. """
@@ -79,15 +77,3 @@ def _load_file_contents(path):
   """ Returns the contents of the specified file path. """
   with open(path) as f:
     return f.read()
-
-
-InstanceKeysContext = namedtuple('InstanceKeysContext', ['instance_key_service',
-                                                         'service_key_expiration',
-                                                         'service_key_kid_location',
-                                                         'service_key_location'])
-
-def instance_keys_context_from_app_config(app_config):
-  return InstanceKeysContext(app_config['INSTANCE_SERVICE_KEY_SERVICE'],
-                             app_config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120),
-                             app_config['INSTANCE_SERVICE_KEY_KID_LOCATION'],
-                             app_config['INSTANCE_SERVICE_KEY_LOCATION'])

util/tufmetadata/api.py

@@ -11,7 +11,7 @@ import requests
 from data.database import CloseForLongOperation
 from util.abchelpers import nooper
 from util.failover import failover, FailoverException
-from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
+from util.security.instancekeys import InstanceKeys
 from util.security.registry_jwt import (build_context_and_subject, generate_bearer_token,
                                         SIGNER_TUF_ROOT)
 
@@ -108,7 +108,7 @@ class NoopTUFMetadataAPI(TUFMetadataAPIInterface):
 class ImplementedTUFMetadataAPI(TUFMetadataAPIInterface):
   def __init__(self, app, config, client=None):
     self._app = app
-    self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config))
+    self._instance_keys = InstanceKeys(app)
     self._config = config
     self._client = client or config['HTTPCLIENT']
     self._gun_prefix = config['TUF_GUN_PREFIX'] or config['SERVER_HOSTNAME']

workers/securityworker/securityworker.py

@@ -19,7 +19,7 @@ DEFAULT_INDEXING_INTERVAL = 30
 class SecurityWorker(Worker):
   def __init__(self):
     super(SecurityWorker, self).__init__()
-    validator = SecurityConfigValidator(app.config)
+    validator = SecurityConfigValidator(app.config.get('FEATURE_SECURITY_SCANNER', False), app.config.get('SECURITY_SCANNER_ENDPOINT'))
     if not validator.valid():
       logger.warning('Failed to validate security scan configuration')
       return