Extract the config provider into its own sub-module

app.py

@@ -32,7 +32,8 @@ from util.config.oauth import (GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuth
 from util.security.signing import Signer
 from util.saas.cloudwatch import start_cloudwatch_sender
 from util.saas.metricqueue import MetricQueue
-from util.config.provider import FileConfigProvider, TestConfigProvider
+from util.saas.queuemetrics import QueueMetrics
+from util.config.provider import get_config_provider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
 
@@ -42,8 +43,6 @@ OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py'
 
 OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG'
 
-CONFIG_PROVIDER = FileConfigProvider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py')
-
 app = Flask(__name__)
 logger = logging.getLogger(__name__)
 
@@ -57,9 +56,11 @@ class RegexConverter(BaseConverter):
 app.url_map.converters['regex'] = RegexConverter
 
 # Instantiate the default configuration (for test or for normal operation).
-if 'TEST' in os.environ:
-  CONFIG_PROVIDER = TestConfigProvider()
+is_testing = 'TEST' in os.environ
+config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py',
+                                      testing=is_testing)
 
+if is_testing:
   from test.testconfig import TestConfig
   logger.debug('Loading test config.')
   app.config.from_object(TestConfig())
@@ -70,7 +71,7 @@ else:
   app.teardown_request(database.close_db_filter)
 
 # Load the override config via the provider.
-CONFIG_PROVIDER.update_app_config(app.config)
+config_provider.update_app_config(app.config)
 
 # Update any configuration found in the override environment variable.
 OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG'

endpoints/api/suconfig.py

@@ -9,7 +9,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_
                            require_fresh_login, request, validate_json_request, verify_not_prod)
 
 from endpoints.common import common_login
-from app import app, CONFIG_PROVIDER, superusers
+from app import app, config_provider, superusers
 from data import model
 from data.database import configure
 from auth.permissions import SuperUserPermission
@@ -56,13 +56,13 @@ class SuperUserRegistryStatus(ApiResource):
     """ Returns the status of the registry. """
 
     # If there is no conf/stack volume, then report that status.
-    if not CONFIG_PROVIDER.volume_exists():
+    if not config_provider.volume_exists():
       return {
         'status': 'missing-config-dir'
       }
 
     # If there is no config file, we need to setup the database.
-    if not CONFIG_PROVIDER.yaml_exists():
+    if not config_provider.yaml_exists():
       return {
         'status': 'config-db'
       }
@@ -76,7 +76,7 @@ class SuperUserRegistryStatus(ApiResource):
     # If we have SETUP_COMPLETE, then we're ready to go!
     if app.config.get('SETUP_COMPLETE', False):
       return {
-        'requires_restart': CONFIG_PROVIDER.requires_restart(app.config),
+        'requires_restart': config_provider.requires_restart(app.config),
         'status': 'ready'
       }
 
@@ -107,10 +107,10 @@ class SuperUserSetupDatabase(ApiResource):
     """ Invokes the alembic upgrade process. """
     # Note: This method is called after the database configured is saved, but before the
     # database has any tables. Therefore, we only allow it to be run in that unique case.
-    if CONFIG_PROVIDER.yaml_exists() and not database_is_valid():
+    if config_provider.yaml_exists() and not database_is_valid():
       # Note: We need to reconfigure the database here as the config has changed.
       combined = dict(**app.config)
-      combined.update(CONFIG_PROVIDER.get_yaml())
+      combined.update(config_provider.get_yaml())
 
       configure(combined)
       app.config['DB_URI'] = combined['DB_URI']
@@ -185,7 +185,7 @@ class SuperUserConfig(ApiResource):
   def get(self):
     """ Returns the currently defined configuration, if any. """
     if SuperUserPermission().can():
-      config_object = CONFIG_PROVIDER.get_yaml()
+      config_object = config_provider.get_yaml()
       return {
         'config': config_object
       }
@@ -199,7 +199,7 @@ class SuperUserConfig(ApiResource):
     """ Updates the config.yaml file. """
     # Note: This method is called to set the database configuration before super users exists,
     # so we also allow it to be called if there is no valid registry configuration setup.
-    if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
+    if not config_provider.yaml_exists() or SuperUserPermission().can():
       config_object = request.get_json()['config']
       hostname = request.get_json()['hostname']
 
@@ -207,7 +207,7 @@ class SuperUserConfig(ApiResource):
       add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname)
 
       # Write the configuration changes to the YAML file.
-      CONFIG_PROVIDER.save_yaml(config_object)
+      config_provider.save_yaml(config_object)
 
       # If the authentication system is not the database, link the superuser account to the
       # the authentication system chosen.
@@ -238,7 +238,7 @@ class SuperUserConfigFile(ApiResource):
 
     if SuperUserPermission().can():
       return {
-       'exists': CONFIG_PROVIDER.volume_file_exists(filename)
+       'exists': config_provider.volume_file_exists(filename)
       }
 
     abort(403)
@@ -252,12 +252,12 @@ class SuperUserConfigFile(ApiResource):
 
     # Note: This method can be called before the configuration exists
     # to upload the database SSL cert.
-    if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
+    if not config_provider.yaml_exists() or SuperUserPermission().can():
       uploaded_file = request.files['file']
       if not uploaded_file:
         abort(400)
 
-      CONFIG_PROVIDER.save_volume_file(filename, uploaded_file)
+      config_provider.save_volume_file(filename, uploaded_file)
       return {
         'status': True
       }
@@ -309,7 +309,7 @@ class SuperUserCreateInitialSuperUser(ApiResource):
     #
     # We do this special security check because at the point this method is called, the database
     # is clean but does not (yet) have any super users for our permissions code to check against.
-    if CONFIG_PROVIDER.yaml_exists() and not database_has_users():
+    if config_provider.yaml_exists() and not database_has_users():
       data = request.get_json()
       username = data['username']
       password = data['password']
@@ -319,9 +319,9 @@ class SuperUserCreateInitialSuperUser(ApiResource):
       superuser = model.user.create_user(username, password, email, auto_verify=True)
 
       # Add the user to the config.
-      config_object = CONFIG_PROVIDER.get_yaml()
+      config_object = config_provider.get_yaml()
       config_object['SUPER_USERS'] = [username]
-      CONFIG_PROVIDER.save_yaml(config_object)
+      config_provider.save_yaml(config_object)
 
       # Update the in-memory config for the new superuser.
       superusers.register_superuser(username)
@@ -369,7 +369,7 @@ class SuperUserConfigValidate(ApiResource):
     # Note: This method is called to validate the database configuration before super users exists,
     # so we also allow it to be called if there is no valid registry configuration setup. Note that
     # this is also safe since this method does not access any information not given in the request.
-    if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
+    if not config_provider.yaml_exists() or SuperUserPermission().can():
       config = request.get_json()['config']
       return validate_service_for_config(service, config, request.get_json().get('password', ''))
 

endpoints/decorated.py

@@ -4,7 +4,7 @@ import json
 from flask import make_response
 from app import app
 from util.useremails import CannotSendEmailException
-from util.config.provider import CannotWriteConfigException
+from util.config.provider.baseprovider import CannotWriteConfigException
 from data import model
 
 logger = logging.getLogger(__name__)

test/test_suconfig_api.py

@@ -1,7 +1,7 @@
 from test.test_api_usage import ApiTestCase, READ_ACCESS_USER, ADMIN_ACCESS_USER
 from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile,
                                     SuperUserCreateInitialSuperUser, SuperUserConfigValidate)
-from app import CONFIG_PROVIDER
+from app import config_provider
 from data.database import User
 
 import unittest
@@ -10,11 +10,11 @@ import unittest
 class ConfigForTesting(object):
 
   def __enter__(self):
-    CONFIG_PROVIDER.reset_for_test()
-    return CONFIG_PROVIDER
+    config_provider.reset_for_test()
+    return config_provider
 
   def __exit__(self, type, value, traceback):
-    CONFIG_PROVIDER.reset_for_test()
+    config_provider.reset_for_test()
 
 
 class TestSuperUserRegistryStatus(ApiTestCase):

util/config/provider.py

@@ -1,181 +0,0 @@
-import os
-import yaml
-import logging
-import json
-from StringIO import StringIO
-
-logger = logging.getLogger(__name__)
-
-class CannotWriteConfigException(Exception):
-  """ Exception raised when the config cannot be written. """
-  pass
-
-def _import_yaml(config_obj, config_file):
-  with open(config_file) as f:
-    c = yaml.safe_load(f)
-    if not c:
-      logger.debug('Empty YAML config file')
-      return
-
-    if isinstance(c, str):
-      raise Exception('Invalid YAML config file: ' + str(c))
-
-    for key in c.iterkeys():
-      if key.isupper():
-        config_obj[key] = c[key]
-
-  return config_obj
-
-
-def _export_yaml(config_obj, config_file):
-  try:
-    with open(config_file, 'w') as f:
-      f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True))
-  except IOError as ioe:
-    raise CannotWriteConfigException(str(ioe))
-
-
-class BaseProvider(object):
-  """ A configuration provider helps to load, save, and handle config override in the application.
-  """
-
-  def update_app_config(self, app_config):
-    """ Updates the given application config object with the loaded override config. """
-    raise NotImplementedError
-
-  def get_yaml(self):
-    """ Returns the contents of the YAML config override file, or None if none. """
-    raise NotImplementedError
-
-  def save_yaml(self, config_object):
-    """ Updates the contents of the YAML config override file to those given. """
-    raise NotImplementedError
-
-  def yaml_exists(self):
-    """ Returns true if a YAML config override file exists in the config volume. """
-    raise NotImplementedError
-
-  def volume_exists(self):
-    """ Returns whether the config override volume exists. """
-    raise NotImplementedError
-
-  def volume_file_exists(self, filename):
-    """ Returns whether the file with the given name exists under the config override volume. """
-    raise NotImplementedError
-
-  def get_volume_file(self, filename, mode='r'):
-    """ Returns a Python file referring to the given name under the config override volumne. """
-    raise NotImplementedError
-
-  def save_volume_file(self, filename, flask_file):
-    """ Saves the given flask file to the config override volume, with the given
-        filename.
-    """
-    raise NotImplementedError
-
-  def requires_restart(self, app_config):
-    """ If true, the configuration loaded into memory for the app does not match that on disk,
-        indicating that this container requires a restart.
-    """
-    raise NotImplementedError
-
-
-class FileConfigProvider(BaseProvider):
-  """ Implementation of the config provider that reads the data from the file system. """
-  def __init__(self, config_volume, yaml_filename, py_filename):
-    self.config_volume = config_volume
-    self.yaml_filename = yaml_filename
-    self.py_filename = py_filename
-
-    self.yaml_path = os.path.join(config_volume, yaml_filename)
-    self.py_path = os.path.join(config_volume, py_filename)
-
-  def update_app_config(self, app_config):
-    if os.path.exists(self.py_path):
-      logger.debug('Applying config file: %s', self.py_path)
-      app_config.from_pyfile(self.py_path)
-
-    if os.path.exists(self.yaml_path):
-      logger.debug('Applying config file: %s', self.yaml_path)
-      _import_yaml(app_config, self.yaml_path)
-
-  def get_yaml(self):
-    if not os.path.exists(self.yaml_path):
-      return None
-
-    config_obj = {}
-    _import_yaml(config_obj, self.yaml_path)
-    return config_obj
-
-  def save_yaml(self, config_obj):
-    _export_yaml(config_obj, self.yaml_path)
-
-  def yaml_exists(self):
-    return self.volume_file_exists(self.yaml_filename)
-
-  def volume_exists(self):
-    return os.path.exists(self.config_volume)
-
-  def volume_file_exists(self, filename):
-    return os.path.exists(os.path.join(self.config_volume, filename))
-
-  def get_volume_file(self, filename, mode='r'):
-    return open(os.path.join(self.config_volume, filename), mode)
-
-  def save_volume_file(self, filename, flask_file):
-    try:
-      flask_file.save(os.path.join(self.config_volume, filename))
-    except IOError as ioe:
-      raise CannotWriteConfigException(str(ioe))
-
-  def requires_restart(self, app_config):
-    file_config = self.get_yaml()
-    if not file_config:
-      return False
-
-    for key in file_config:
-      if app_config.get(key) != file_config[key]:
-        return True
-
-    return False
-
-class TestConfigProvider(BaseProvider):
-  """ Implementation of the config provider for testing. Everything is kept in-memory instead on
-      the real file system. """
-  def __init__(self):
-    self.files = {}
-    self._config = None
-
-  def update_app_config(self, app_config):
-    self._config = app_config
-
-  def get_yaml(self):
-    if not 'config.yaml' in self.files:
-      return None
-
-    return json.loads(self.files.get('config.yaml', '{}'))
-
-  def save_yaml(self, config_obj):
-    self.files['config.yaml'] = json.dumps(config_obj)
-
-  def yaml_exists(self):
-    return 'config.yaml' in self.files
-
-  def volume_exists(self):
-    return True
-
-  def volume_file_exists(self, filename):
-    return filename in self.files
-
-  def save_volume_file(self, filename, flask_file):
-    self.files[filename] = ''
-
-  def get_volume_file(self, filename, mode='r'):
-    return StringIO(self.files[filename])
-
-  def requires_restart(self, app_config):
-    return False
-
-  def reset_for_test(self):
-    self._config['SUPER_USERS'] = ['devtable']
-    self.files = {}

util/config/provider/__init__.py

@@ -0,0 +1,10 @@
+from util.config.provider.fileprovider import FileConfigProvider
+from util.config.provider.testprovider import TestConfigProvider
+
+def get_config_provider(config_volume, yaml_filename, py_filename, testing=False):
+  """ Loads and returns the config provider for the current environment. """
+  if testing:
+    return TestConfigProvider()
+
+  return FileConfigProvider(config_volume, yaml_filename, py_filename)
+

util/config/provider/baseprovider.py

@@ -0,0 +1,77 @@
+import yaml
+import logging
+
+logger = logging.getLogger(__name__)
+
+class CannotWriteConfigException(Exception):
+  """ Exception raised when the config cannot be written. """
+  pass
+
+def import_yaml(config_obj, config_file):
+  with open(config_file) as f:
+    c = yaml.safe_load(f)
+    if not c:
+      logger.debug('Empty YAML config file')
+      return
+
+    if isinstance(c, str):
+      raise Exception('Invalid YAML config file: ' + str(c))
+
+    for key in c.iterkeys():
+      if key.isupper():
+        config_obj[key] = c[key]
+
+  return config_obj
+
+
+def export_yaml(config_obj, config_file):
+  try:
+    with open(config_file, 'w') as f:
+      f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True))
+  except IOError as ioe:
+    raise CannotWriteConfigException(str(ioe))
+
+
+class BaseProvider(object):
+  """ A configuration provider helps to load, save, and handle config override in the application.
+  """
+
+  def update_app_config(self, app_config):
+    """ Updates the given application config object with the loaded override config. """
+    raise NotImplementedError
+
+  def get_yaml(self):
+    """ Returns the contents of the YAML config override file, or None if none. """
+    raise NotImplementedError
+
+  def save_yaml(self, config_object):
+    """ Updates the contents of the YAML config override file to those given. """
+    raise NotImplementedError
+
+  def yaml_exists(self):
+    """ Returns true if a YAML config override file exists in the config volume. """
+    raise NotImplementedError
+
+  def volume_exists(self):
+    """ Returns whether the config override volume exists. """
+    raise NotImplementedError
+
+  def volume_file_exists(self, filename):
+    """ Returns whether the file with the given name exists under the config override volume. """
+    raise NotImplementedError
+
+  def get_volume_file(self, filename, mode='r'):
+    """ Returns a Python file referring to the given name under the config override volumne. """
+    raise NotImplementedError
+
+  def save_volume_file(self, filename, flask_file):
+    """ Saves the given flask file to the config override volume, with the given
+        filename.
+    """
+    raise NotImplementedError
+
+  def requires_restart(self, app_config):
+    """ If true, the configuration loaded into memory for the app does not match that on disk,
+        indicating that this container requires a restart.
+    """
+    raise NotImplementedError

util/config/provider/fileprovider.py

@@ -0,0 +1,66 @@
+import os
+import logging
+
+from util.config.provider.baseprovider import (BaseProvider, import_yaml, export_yaml,
+                                               CannotWriteConfigException)
+
+logger = logging.getLogger(__name__)
+
+class FileConfigProvider(BaseProvider):
+  """ Implementation of the config provider that reads the data from the file system. """
+  def __init__(self, config_volume, yaml_filename, py_filename):
+    self.config_volume = config_volume
+    self.yaml_filename = yaml_filename
+    self.py_filename = py_filename
+
+    self.yaml_path = os.path.join(config_volume, yaml_filename)
+    self.py_path = os.path.join(config_volume, py_filename)
+
+  def update_app_config(self, app_config):
+    if os.path.exists(self.py_path):
+      logger.debug('Applying config file: %s', self.py_path)
+      app_config.from_pyfile(self.py_path)
+
+    if os.path.exists(self.yaml_path):
+      logger.debug('Applying config file: %s', self.yaml_path)
+      import_yaml(app_config, self.yaml_path)
+
+  def get_yaml(self):
+    if not os.path.exists(self.yaml_path):
+      return None
+
+    config_obj = {}
+    import_yaml(config_obj, self.yaml_path)
+    return config_obj
+
+  def save_yaml(self, config_obj):
+    export_yaml(config_obj, self.yaml_path)
+
+  def yaml_exists(self):
+    return self.volume_file_exists(self.yaml_filename)
+
+  def volume_exists(self):
+    return os.path.exists(self.config_volume)
+
+  def volume_file_exists(self, filename):
+    return os.path.exists(os.path.join(self.config_volume, filename))
+
+  def get_volume_file(self, filename, mode='r'):
+    return open(os.path.join(self.config_volume, filename), mode)
+
+  def save_volume_file(self, filename, flask_file):
+    try:
+      flask_file.save(os.path.join(self.config_volume, filename))
+    except IOError as ioe:
+      raise CannotWriteConfigException(str(ioe))
+
+  def requires_restart(self, app_config):
+    file_config = self.get_yaml()
+    if not file_config:
+      return False
+
+    for key in file_config:
+      if app_config.get(key) != file_config[key]:
+        return True
+
+    return False

util/config/provider/testprovider.py

@@ -0,0 +1,45 @@
+import json
+from StringIO import StringIO
+
+from util.config.provider.baseprovider import BaseProvider
+
+class TestConfigProvider(BaseProvider):
+  """ Implementation of the config provider for testing. Everything is kept in-memory instead on
+      the real file system. """
+  def __init__(self):
+    self.files = {}
+    self._config = None
+
+  def update_app_config(self, app_config):
+    self._config = app_config
+
+  def get_yaml(self):
+    if not 'config.yaml' in self.files:
+      return None
+
+    return json.loads(self.files.get('config.yaml', '{}'))
+
+  def save_yaml(self, config_obj):
+    self.files['config.yaml'] = json.dumps(config_obj)
+
+  def yaml_exists(self):
+    return 'config.yaml' in self.files
+
+  def volume_exists(self):
+    return True
+
+  def volume_file_exists(self, filename):
+    return filename in self.files
+
+  def save_volume_file(self, filename, flask_file):
+    self.files[filename] = ''
+
+  def get_volume_file(self, filename, mode='r'):
+    return StringIO(self.files[filename])
+
+  def requires_restart(self, app_config):
+    return False
+
+  def reset_for_test(self):
+    self._config['SUPER_USERS'] = ['devtable']
+    self.files = {}

util/config/validator.py

@@ -19,7 +19,7 @@ from auth.auth_context import get_authenticated_user
 from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
 from bitbucket import BitBucket
 
-from app import app, CONFIG_PROVIDER, get_app_url, OVERRIDE_CONFIG_DIRECTORY
+from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
 
 logger = logging.getLogger(__name__)
 
@@ -223,10 +223,10 @@ def _validate_ssl(config, _):
     return
 
   for filename in SSL_FILENAMES:
-    if not CONFIG_PROVIDER.volume_file_exists(filename):
+    if not config_provider.volume_file_exists(filename):
       raise Exception('Missing required SSL file: %s' % filename)
 
-  with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[0]) as f:
+  with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
     cert_contents = f.read()
 
   # Validate the certificate.
@@ -239,7 +239,7 @@ def _validate_ssl(config, _):
     raise Exception('The specified SSL certificate has expired.')
 
   private_key_path = None
-  with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[1]) as f:
+  with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
     private_key_path = f.name
 
   if not private_key_path: