diff --git a/app.py b/app.py index 68e527ee2..cdbc7d08a 100644 --- a/app.py +++ b/app.py @@ -32,7 +32,8 @@ from util.config.oauth import (GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuth from util.security.signing import Signer from util.saas.cloudwatch import start_cloudwatch_sender from util.saas.metricqueue import MetricQueue -from util.config.provider import FileConfigProvider, TestConfigProvider +from util.saas.queuemetrics import QueueMetrics +from util.config.provider import get_config_provider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager @@ -42,8 +43,6 @@ OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py' OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' -CONFIG_PROVIDER = FileConfigProvider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py') - app = Flask(__name__) logger = logging.getLogger(__name__) @@ -57,9 +56,11 @@ class RegexConverter(BaseConverter): app.url_map.converters['regex'] = RegexConverter # Instantiate the default configuration (for test or for normal operation). -if 'TEST' in os.environ: - CONFIG_PROVIDER = TestConfigProvider() +is_testing = 'TEST' in os.environ +config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py', + testing=is_testing) +if is_testing: from test.testconfig import TestConfig logger.debug('Loading test config.') app.config.from_object(TestConfig()) @@ -70,7 +71,7 @@ else: app.teardown_request(database.close_db_filter) # Load the override config via the provider. -CONFIG_PROVIDER.update_app_config(app.config) +config_provider.update_app_config(app.config) # Update any configuration found in the override environment variable. OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 1b24da70c..a4edf6839 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -9,7 +9,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_ require_fresh_login, request, validate_json_request, verify_not_prod) from endpoints.common import common_login -from app import app, CONFIG_PROVIDER, superusers +from app import app, config_provider, superusers from data import model from data.database import configure from auth.permissions import SuperUserPermission @@ -56,13 +56,13 @@ class SuperUserRegistryStatus(ApiResource): """ Returns the status of the registry. """ # If there is no conf/stack volume, then report that status. - if not CONFIG_PROVIDER.volume_exists(): + if not config_provider.volume_exists(): return { 'status': 'missing-config-dir' } # If there is no config file, we need to setup the database. - if not CONFIG_PROVIDER.yaml_exists(): + if not config_provider.yaml_exists(): return { 'status': 'config-db' } @@ -76,7 +76,7 @@ class SuperUserRegistryStatus(ApiResource): # If we have SETUP_COMPLETE, then we're ready to go! if app.config.get('SETUP_COMPLETE', False): return { - 'requires_restart': CONFIG_PROVIDER.requires_restart(app.config), + 'requires_restart': config_provider.requires_restart(app.config), 'status': 'ready' } @@ -107,10 +107,10 @@ class SuperUserSetupDatabase(ApiResource): """ Invokes the alembic upgrade process. """ # Note: This method is called after the database configured is saved, but before the # database has any tables. Therefore, we only allow it to be run in that unique case. - if CONFIG_PROVIDER.yaml_exists() and not database_is_valid(): + if config_provider.yaml_exists() and not database_is_valid(): # Note: We need to reconfigure the database here as the config has changed. combined = dict(**app.config) - combined.update(CONFIG_PROVIDER.get_yaml()) + combined.update(config_provider.get_yaml()) configure(combined) app.config['DB_URI'] = combined['DB_URI'] @@ -185,7 +185,7 @@ class SuperUserConfig(ApiResource): def get(self): """ Returns the currently defined configuration, if any. """ if SuperUserPermission().can(): - config_object = CONFIG_PROVIDER.get_yaml() + config_object = config_provider.get_yaml() return { 'config': config_object } @@ -199,7 +199,7 @@ class SuperUserConfig(ApiResource): """ Updates the config.yaml file. """ # Note: This method is called to set the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. - if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can(): + if not config_provider.yaml_exists() or SuperUserPermission().can(): config_object = request.get_json()['config'] hostname = request.get_json()['hostname'] @@ -207,7 +207,7 @@ class SuperUserConfig(ApiResource): add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname) # Write the configuration changes to the YAML file. - CONFIG_PROVIDER.save_yaml(config_object) + config_provider.save_yaml(config_object) # If the authentication system is not the database, link the superuser account to the # the authentication system chosen. @@ -238,7 +238,7 @@ class SuperUserConfigFile(ApiResource): if SuperUserPermission().can(): return { - 'exists': CONFIG_PROVIDER.volume_file_exists(filename) + 'exists': config_provider.volume_file_exists(filename) } abort(403) @@ -252,12 +252,12 @@ class SuperUserConfigFile(ApiResource): # Note: This method can be called before the configuration exists # to upload the database SSL cert. - if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can(): + if not config_provider.yaml_exists() or SuperUserPermission().can(): uploaded_file = request.files['file'] if not uploaded_file: abort(400) - CONFIG_PROVIDER.save_volume_file(filename, uploaded_file) + config_provider.save_volume_file(filename, uploaded_file) return { 'status': True } @@ -309,7 +309,7 @@ class SuperUserCreateInitialSuperUser(ApiResource): # # We do this special security check because at the point this method is called, the database # is clean but does not (yet) have any super users for our permissions code to check against. - if CONFIG_PROVIDER.yaml_exists() and not database_has_users(): + if config_provider.yaml_exists() and not database_has_users(): data = request.get_json() username = data['username'] password = data['password'] @@ -319,9 +319,9 @@ class SuperUserCreateInitialSuperUser(ApiResource): superuser = model.user.create_user(username, password, email, auto_verify=True) # Add the user to the config. - config_object = CONFIG_PROVIDER.get_yaml() + config_object = config_provider.get_yaml() config_object['SUPER_USERS'] = [username] - CONFIG_PROVIDER.save_yaml(config_object) + config_provider.save_yaml(config_object) # Update the in-memory config for the new superuser. superusers.register_superuser(username) @@ -369,7 +369,7 @@ class SuperUserConfigValidate(ApiResource): # Note: This method is called to validate the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. Note that # this is also safe since this method does not access any information not given in the request. - if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can(): + if not config_provider.yaml_exists() or SuperUserPermission().can(): config = request.get_json()['config'] return validate_service_for_config(service, config, request.get_json().get('password', '')) diff --git a/endpoints/decorated.py b/endpoints/decorated.py index b51e1ee2e..d1e654da0 100644 --- a/endpoints/decorated.py +++ b/endpoints/decorated.py @@ -4,7 +4,7 @@ import json from flask import make_response from app import app from util.useremails import CannotSendEmailException -from util.config.provider import CannotWriteConfigException +from util.config.provider.baseprovider import CannotWriteConfigException from data import model logger = logging.getLogger(__name__) diff --git a/test/test_suconfig_api.py b/test/test_suconfig_api.py index 7e049f610..795ab8fe5 100644 --- a/test/test_suconfig_api.py +++ b/test/test_suconfig_api.py @@ -1,7 +1,7 @@ from test.test_api_usage import ApiTestCase, READ_ACCESS_USER, ADMIN_ACCESS_USER from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile, SuperUserCreateInitialSuperUser, SuperUserConfigValidate) -from app import CONFIG_PROVIDER +from app import config_provider from data.database import User import unittest @@ -10,11 +10,11 @@ import unittest class ConfigForTesting(object): def __enter__(self): - CONFIG_PROVIDER.reset_for_test() - return CONFIG_PROVIDER + config_provider.reset_for_test() + return config_provider def __exit__(self, type, value, traceback): - CONFIG_PROVIDER.reset_for_test() + config_provider.reset_for_test() class TestSuperUserRegistryStatus(ApiTestCase): diff --git a/util/config/provider.py b/util/config/provider.py deleted file mode 100644 index 32bc4c6f9..000000000 --- a/util/config/provider.py +++ /dev/null @@ -1,181 +0,0 @@ -import os -import yaml -import logging -import json -from StringIO import StringIO - -logger = logging.getLogger(__name__) - -class CannotWriteConfigException(Exception): - """ Exception raised when the config cannot be written. """ - pass - -def _import_yaml(config_obj, config_file): - with open(config_file) as f: - c = yaml.safe_load(f) - if not c: - logger.debug('Empty YAML config file') - return - - if isinstance(c, str): - raise Exception('Invalid YAML config file: ' + str(c)) - - for key in c.iterkeys(): - if key.isupper(): - config_obj[key] = c[key] - - return config_obj - - -def _export_yaml(config_obj, config_file): - try: - with open(config_file, 'w') as f: - f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)) - except IOError as ioe: - raise CannotWriteConfigException(str(ioe)) - - -class BaseProvider(object): - """ A configuration provider helps to load, save, and handle config override in the application. - """ - - def update_app_config(self, app_config): - """ Updates the given application config object with the loaded override config. """ - raise NotImplementedError - - def get_yaml(self): - """ Returns the contents of the YAML config override file, or None if none. """ - raise NotImplementedError - - def save_yaml(self, config_object): - """ Updates the contents of the YAML config override file to those given. """ - raise NotImplementedError - - def yaml_exists(self): - """ Returns true if a YAML config override file exists in the config volume. """ - raise NotImplementedError - - def volume_exists(self): - """ Returns whether the config override volume exists. """ - raise NotImplementedError - - def volume_file_exists(self, filename): - """ Returns whether the file with the given name exists under the config override volume. """ - raise NotImplementedError - - def get_volume_file(self, filename, mode='r'): - """ Returns a Python file referring to the given name under the config override volumne. """ - raise NotImplementedError - - def save_volume_file(self, filename, flask_file): - """ Saves the given flask file to the config override volume, with the given - filename. - """ - raise NotImplementedError - - def requires_restart(self, app_config): - """ If true, the configuration loaded into memory for the app does not match that on disk, - indicating that this container requires a restart. - """ - raise NotImplementedError - - -class FileConfigProvider(BaseProvider): - """ Implementation of the config provider that reads the data from the file system. """ - def __init__(self, config_volume, yaml_filename, py_filename): - self.config_volume = config_volume - self.yaml_filename = yaml_filename - self.py_filename = py_filename - - self.yaml_path = os.path.join(config_volume, yaml_filename) - self.py_path = os.path.join(config_volume, py_filename) - - def update_app_config(self, app_config): - if os.path.exists(self.py_path): - logger.debug('Applying config file: %s', self.py_path) - app_config.from_pyfile(self.py_path) - - if os.path.exists(self.yaml_path): - logger.debug('Applying config file: %s', self.yaml_path) - _import_yaml(app_config, self.yaml_path) - - def get_yaml(self): - if not os.path.exists(self.yaml_path): - return None - - config_obj = {} - _import_yaml(config_obj, self.yaml_path) - return config_obj - - def save_yaml(self, config_obj): - _export_yaml(config_obj, self.yaml_path) - - def yaml_exists(self): - return self.volume_file_exists(self.yaml_filename) - - def volume_exists(self): - return os.path.exists(self.config_volume) - - def volume_file_exists(self, filename): - return os.path.exists(os.path.join(self.config_volume, filename)) - - def get_volume_file(self, filename, mode='r'): - return open(os.path.join(self.config_volume, filename), mode) - - def save_volume_file(self, filename, flask_file): - try: - flask_file.save(os.path.join(self.config_volume, filename)) - except IOError as ioe: - raise CannotWriteConfigException(str(ioe)) - - def requires_restart(self, app_config): - file_config = self.get_yaml() - if not file_config: - return False - - for key in file_config: - if app_config.get(key) != file_config[key]: - return True - - return False - -class TestConfigProvider(BaseProvider): - """ Implementation of the config provider for testing. Everything is kept in-memory instead on - the real file system. """ - def __init__(self): - self.files = {} - self._config = None - - def update_app_config(self, app_config): - self._config = app_config - - def get_yaml(self): - if not 'config.yaml' in self.files: - return None - - return json.loads(self.files.get('config.yaml', '{}')) - - def save_yaml(self, config_obj): - self.files['config.yaml'] = json.dumps(config_obj) - - def yaml_exists(self): - return 'config.yaml' in self.files - - def volume_exists(self): - return True - - def volume_file_exists(self, filename): - return filename in self.files - - def save_volume_file(self, filename, flask_file): - self.files[filename] = '' - - def get_volume_file(self, filename, mode='r'): - return StringIO(self.files[filename]) - - def requires_restart(self, app_config): - return False - - def reset_for_test(self): - self._config['SUPER_USERS'] = ['devtable'] - self.files = {} diff --git a/util/config/provider/__init__.py b/util/config/provider/__init__.py new file mode 100644 index 000000000..4445d696c --- /dev/null +++ b/util/config/provider/__init__.py @@ -0,0 +1,10 @@ +from util.config.provider.fileprovider import FileConfigProvider +from util.config.provider.testprovider import TestConfigProvider + +def get_config_provider(config_volume, yaml_filename, py_filename, testing=False): + """ Loads and returns the config provider for the current environment. """ + if testing: + return TestConfigProvider() + + return FileConfigProvider(config_volume, yaml_filename, py_filename) + diff --git a/util/config/provider/baseprovider.py b/util/config/provider/baseprovider.py new file mode 100644 index 000000000..10a63b7c9 --- /dev/null +++ b/util/config/provider/baseprovider.py @@ -0,0 +1,77 @@ +import yaml +import logging + +logger = logging.getLogger(__name__) + +class CannotWriteConfigException(Exception): + """ Exception raised when the config cannot be written. """ + pass + +def import_yaml(config_obj, config_file): + with open(config_file) as f: + c = yaml.safe_load(f) + if not c: + logger.debug('Empty YAML config file') + return + + if isinstance(c, str): + raise Exception('Invalid YAML config file: ' + str(c)) + + for key in c.iterkeys(): + if key.isupper(): + config_obj[key] = c[key] + + return config_obj + + +def export_yaml(config_obj, config_file): + try: + with open(config_file, 'w') as f: + f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)) + except IOError as ioe: + raise CannotWriteConfigException(str(ioe)) + + +class BaseProvider(object): + """ A configuration provider helps to load, save, and handle config override in the application. + """ + + def update_app_config(self, app_config): + """ Updates the given application config object with the loaded override config. """ + raise NotImplementedError + + def get_yaml(self): + """ Returns the contents of the YAML config override file, or None if none. """ + raise NotImplementedError + + def save_yaml(self, config_object): + """ Updates the contents of the YAML config override file to those given. """ + raise NotImplementedError + + def yaml_exists(self): + """ Returns true if a YAML config override file exists in the config volume. """ + raise NotImplementedError + + def volume_exists(self): + """ Returns whether the config override volume exists. """ + raise NotImplementedError + + def volume_file_exists(self, filename): + """ Returns whether the file with the given name exists under the config override volume. """ + raise NotImplementedError + + def get_volume_file(self, filename, mode='r'): + """ Returns a Python file referring to the given name under the config override volumne. """ + raise NotImplementedError + + def save_volume_file(self, filename, flask_file): + """ Saves the given flask file to the config override volume, with the given + filename. + """ + raise NotImplementedError + + def requires_restart(self, app_config): + """ If true, the configuration loaded into memory for the app does not match that on disk, + indicating that this container requires a restart. + """ + raise NotImplementedError \ No newline at end of file diff --git a/util/config/provider/fileprovider.py b/util/config/provider/fileprovider.py new file mode 100644 index 000000000..2f616aa44 --- /dev/null +++ b/util/config/provider/fileprovider.py @@ -0,0 +1,66 @@ +import os +import logging + +from util.config.provider.baseprovider import (BaseProvider, import_yaml, export_yaml, + CannotWriteConfigException) + +logger = logging.getLogger(__name__) + +class FileConfigProvider(BaseProvider): + """ Implementation of the config provider that reads the data from the file system. """ + def __init__(self, config_volume, yaml_filename, py_filename): + self.config_volume = config_volume + self.yaml_filename = yaml_filename + self.py_filename = py_filename + + self.yaml_path = os.path.join(config_volume, yaml_filename) + self.py_path = os.path.join(config_volume, py_filename) + + def update_app_config(self, app_config): + if os.path.exists(self.py_path): + logger.debug('Applying config file: %s', self.py_path) + app_config.from_pyfile(self.py_path) + + if os.path.exists(self.yaml_path): + logger.debug('Applying config file: %s', self.yaml_path) + import_yaml(app_config, self.yaml_path) + + def get_yaml(self): + if not os.path.exists(self.yaml_path): + return None + + config_obj = {} + import_yaml(config_obj, self.yaml_path) + return config_obj + + def save_yaml(self, config_obj): + export_yaml(config_obj, self.yaml_path) + + def yaml_exists(self): + return self.volume_file_exists(self.yaml_filename) + + def volume_exists(self): + return os.path.exists(self.config_volume) + + def volume_file_exists(self, filename): + return os.path.exists(os.path.join(self.config_volume, filename)) + + def get_volume_file(self, filename, mode='r'): + return open(os.path.join(self.config_volume, filename), mode) + + def save_volume_file(self, filename, flask_file): + try: + flask_file.save(os.path.join(self.config_volume, filename)) + except IOError as ioe: + raise CannotWriteConfigException(str(ioe)) + + def requires_restart(self, app_config): + file_config = self.get_yaml() + if not file_config: + return False + + for key in file_config: + if app_config.get(key) != file_config[key]: + return True + + return False diff --git a/util/config/provider/testprovider.py b/util/config/provider/testprovider.py new file mode 100644 index 000000000..1e237f3b7 --- /dev/null +++ b/util/config/provider/testprovider.py @@ -0,0 +1,45 @@ +import json +from StringIO import StringIO + +from util.config.provider.baseprovider import BaseProvider + +class TestConfigProvider(BaseProvider): + """ Implementation of the config provider for testing. Everything is kept in-memory instead on + the real file system. """ + def __init__(self): + self.files = {} + self._config = None + + def update_app_config(self, app_config): + self._config = app_config + + def get_yaml(self): + if not 'config.yaml' in self.files: + return None + + return json.loads(self.files.get('config.yaml', '{}')) + + def save_yaml(self, config_obj): + self.files['config.yaml'] = json.dumps(config_obj) + + def yaml_exists(self): + return 'config.yaml' in self.files + + def volume_exists(self): + return True + + def volume_file_exists(self, filename): + return filename in self.files + + def save_volume_file(self, filename, flask_file): + self.files[filename] = '' + + def get_volume_file(self, filename, mode='r'): + return StringIO(self.files[filename]) + + def requires_restart(self, app_config): + return False + + def reset_for_test(self): + self._config['SUPER_USERS'] = ['devtable'] + self.files = {} diff --git a/util/config/validator.py b/util/config/validator.py index 67def6a76..e66cb341d 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -19,7 +19,7 @@ from auth.auth_context import get_authenticated_user from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig from bitbucket import BitBucket -from app import app, CONFIG_PROVIDER, get_app_url, OVERRIDE_CONFIG_DIRECTORY +from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY logger = logging.getLogger(__name__) @@ -223,10 +223,10 @@ def _validate_ssl(config, _): return for filename in SSL_FILENAMES: - if not CONFIG_PROVIDER.volume_file_exists(filename): + if not config_provider.volume_file_exists(filename): raise Exception('Missing required SSL file: %s' % filename) - with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[0]) as f: + with config_provider.get_volume_file(SSL_FILENAMES[0]) as f: cert_contents = f.read() # Validate the certificate. @@ -239,7 +239,7 @@ def _validate_ssl(config, _): raise Exception('The specified SSL certificate has expired.') private_key_path = None - with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[1]) as f: + with config_provider.get_volume_file(SSL_FILENAMES[1]) as f: private_key_path = f.name if not private_key_path: