diff --git a/app.py b/app.py index 68e527ee2..0fcda573c 100644 --- a/app.py +++ b/app.py @@ -32,7 +32,7 @@ from util.config.oauth import (GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuth from util.security.signing import Signer from util.saas.cloudwatch import start_cloudwatch_sender from util.saas.metricqueue import MetricQueue -from util.config.provider import FileConfigProvider, TestConfigProvider +from util.config.provider import get_config_provider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager @@ -42,8 +42,6 @@ OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py' OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' -CONFIG_PROVIDER = FileConfigProvider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py') - app = Flask(__name__) logger = logging.getLogger(__name__) @@ -56,10 +54,13 @@ class RegexConverter(BaseConverter): app.url_map.converters['regex'] = RegexConverter -# Instantiate the default configuration (for test or for normal operation). -if 'TEST' in os.environ: - CONFIG_PROVIDER = TestConfigProvider() +# Instantiate the configuration. +is_testing = 'TEST' in os.environ +is_kubernetes = 'KUBERNETES_SERVICE_HOST' in os.environ +config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py', + testing=is_testing, kubernetes=is_kubernetes) +if is_testing: from test.testconfig import TestConfig logger.debug('Loading test config.') app.config.from_object(TestConfig()) @@ -70,7 +71,7 @@ else: app.teardown_request(database.close_db_filter) # Load the override config via the provider. -CONFIG_PROVIDER.update_app_config(app.config) +config_provider.update_app_config(app.config) # Update any configuration found in the override environment variable. OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 1b24da70c..aaea5a309 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -9,7 +9,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_ require_fresh_login, request, validate_json_request, verify_not_prod) from endpoints.common import common_login -from app import app, CONFIG_PROVIDER, superusers +from app import app, config_provider, superusers from data import model from data.database import configure from auth.permissions import SuperUserPermission @@ -56,13 +56,13 @@ class SuperUserRegistryStatus(ApiResource): """ Returns the status of the registry. """ # If there is no conf/stack volume, then report that status. - if not CONFIG_PROVIDER.volume_exists(): + if not config_provider.volume_exists(): return { 'status': 'missing-config-dir' } # If there is no config file, we need to setup the database. - if not CONFIG_PROVIDER.yaml_exists(): + if not config_provider.config_exists(): return { 'status': 'config-db' } @@ -76,7 +76,7 @@ class SuperUserRegistryStatus(ApiResource): # If we have SETUP_COMPLETE, then we're ready to go! if app.config.get('SETUP_COMPLETE', False): return { - 'requires_restart': CONFIG_PROVIDER.requires_restart(app.config), + 'requires_restart': config_provider.requires_restart(app.config), 'status': 'ready' } @@ -107,10 +107,10 @@ class SuperUserSetupDatabase(ApiResource): """ Invokes the alembic upgrade process. """ # Note: This method is called after the database configured is saved, but before the # database has any tables. Therefore, we only allow it to be run in that unique case. - if CONFIG_PROVIDER.yaml_exists() and not database_is_valid(): + if config_provider.config_exists() and not database_is_valid(): # Note: We need to reconfigure the database here as the config has changed. combined = dict(**app.config) - combined.update(CONFIG_PROVIDER.get_yaml()) + combined.update(config_provider.get_config()) configure(combined) app.config['DB_URI'] = combined['DB_URI'] @@ -185,7 +185,7 @@ class SuperUserConfig(ApiResource): def get(self): """ Returns the currently defined configuration, if any. """ if SuperUserPermission().can(): - config_object = CONFIG_PROVIDER.get_yaml() + config_object = config_provider.get_config() return { 'config': config_object } @@ -196,18 +196,18 @@ class SuperUserConfig(ApiResource): @verify_not_prod @validate_json_request('UpdateConfig') def put(self): - """ Updates the config.yaml file. """ + """ Updates the config override file. """ # Note: This method is called to set the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. - if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can(): + if not config_provider.config_exists() or SuperUserPermission().can(): config_object = request.get_json()['config'] hostname = request.get_json()['hostname'] # Add any enterprise defaults missing from the config. add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname) - # Write the configuration changes to the YAML file. - CONFIG_PROVIDER.save_yaml(config_object) + # Write the configuration changes to the config override file. + config_provider.save_config(config_object) # If the authentication system is not the database, link the superuser account to the # the authentication system chosen. @@ -238,7 +238,7 @@ class SuperUserConfigFile(ApiResource): if SuperUserPermission().can(): return { - 'exists': CONFIG_PROVIDER.volume_file_exists(filename) + 'exists': config_provider.volume_file_exists(filename) } abort(403) @@ -252,12 +252,12 @@ class SuperUserConfigFile(ApiResource): # Note: This method can be called before the configuration exists # to upload the database SSL cert. - if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can(): + if not config_provider.config_exists() or SuperUserPermission().can(): uploaded_file = request.files['file'] if not uploaded_file: abort(400) - CONFIG_PROVIDER.save_volume_file(filename, uploaded_file) + config_provider.save_volume_file(filename, uploaded_file) return { 'status': True } @@ -309,7 +309,7 @@ class SuperUserCreateInitialSuperUser(ApiResource): # # We do this special security check because at the point this method is called, the database # is clean but does not (yet) have any super users for our permissions code to check against. - if CONFIG_PROVIDER.yaml_exists() and not database_has_users(): + if config_provider.config_exists() and not database_has_users(): data = request.get_json() username = data['username'] password = data['password'] @@ -319,9 +319,9 @@ class SuperUserCreateInitialSuperUser(ApiResource): superuser = model.user.create_user(username, password, email, auto_verify=True) # Add the user to the config. - config_object = CONFIG_PROVIDER.get_yaml() + config_object = config_provider.get_config() config_object['SUPER_USERS'] = [username] - CONFIG_PROVIDER.save_yaml(config_object) + config_provider.save_config(config_object) # Update the in-memory config for the new superuser. superusers.register_superuser(username) @@ -369,7 +369,7 @@ class SuperUserConfigValidate(ApiResource): # Note: This method is called to validate the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. Note that # this is also safe since this method does not access any information not given in the request. - if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can(): + if not config_provider.config_exists() or SuperUserPermission().can(): config = request.get_json()['config'] return validate_service_for_config(service, config, request.get_json().get('password', '')) diff --git a/endpoints/decorated.py b/endpoints/decorated.py index b51e1ee2e..d1e654da0 100644 --- a/endpoints/decorated.py +++ b/endpoints/decorated.py @@ -4,7 +4,7 @@ import json from flask import make_response from app import app from util.useremails import CannotSendEmailException -from util.config.provider import CannotWriteConfigException +from util.config.provider.baseprovider import CannotWriteConfigException from data import model logger = logging.getLogger(__name__) diff --git a/endpoints/web.py b/endpoints/web.py index 8018b3666..154483faf 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -9,7 +9,7 @@ from health.healthcheck import get_healthchecker from data import model from data.database import db -from app import app, billing as stripe, build_logs, avatar, signer, log_archive +from app import app, billing as stripe, build_logs, avatar, signer, log_archive, config_provider from auth.auth import require_session_login, process_oauth from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission, SuperUserPermission, AdministerRepositoryPermission, @@ -209,7 +209,7 @@ def v1(): @web.route('/health/instance', methods=['GET']) @no_cache def instance_health(): - checker = get_healthchecker(app) + checker = get_healthchecker(app, config_provider) (data, status_code) = checker.check_instance() response = jsonify(dict(data=data, status_code=status_code)) response.status_code = status_code @@ -221,7 +221,7 @@ def instance_health(): @web.route('/health/endtoend', methods=['GET']) @no_cache def endtoend_health(): - checker = get_healthchecker(app) + checker = get_healthchecker(app, config_provider) (data, status_code) = checker.check_endtoend() response = jsonify(dict(data=data, status_code=status_code)) response.status_code = status_code diff --git a/health/healthcheck.py b/health/healthcheck.py index 98de22435..c212c694d 100644 --- a/health/healthcheck.py +++ b/health/healthcheck.py @@ -4,14 +4,15 @@ from health.services import check_all_services logger = logging.getLogger(__name__) -def get_healthchecker(app): +def get_healthchecker(app, config_provider): """ Returns a HealthCheck instance for the given app. """ - return HealthCheck.get_checker(app) + return HealthCheck.get_checker(app, config_provider) class HealthCheck(object): - def __init__(self, app): + def __init__(self, app, config_provider): self.app = app + self.config_provider = config_provider def check_instance(self): """ @@ -52,20 +53,21 @@ class HealthCheck(object): data = { 'services': service_statuses, 'notes': notes, - 'is_testing': self.app.config['TESTING'] + 'is_testing': self.app.config['TESTING'], + 'config_provider': self.config_provider.provider_id } return (data, 200 if is_healthy else 503) @classmethod - def get_checker(cls, app): + def get_checker(cls, app, config_provider): name = app.config['HEALTH_CHECKER'][0] parameters = app.config['HEALTH_CHECKER'][1] or {} for subc in cls.__subclasses__(): if subc.check_name() == name: - return subc(app, **parameters) + return subc(app, config_provider, **parameters) raise Exception('Unknown health check with name %s' % name) @@ -77,8 +79,8 @@ class LocalHealthCheck(HealthCheck): class ProductionHealthCheck(HealthCheck): - def __init__(self, app, access_key, secret_key, db_instance='quay'): - super(ProductionHealthCheck, self).__init__(app) + def __init__(self, app, config_provider, access_key, secret_key, db_instance='quay'): + super(ProductionHealthCheck, self).__init__(app, config_provider) self.access_key = access_key self.secret_key = secret_key self.db_instance = db_instance diff --git a/test/test_suconfig_api.py b/test/test_suconfig_api.py index 7e049f610..6cbdcafa8 100644 --- a/test/test_suconfig_api.py +++ b/test/test_suconfig_api.py @@ -1,7 +1,7 @@ from test.test_api_usage import ApiTestCase, READ_ACCESS_USER, ADMIN_ACCESS_USER from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, SuperUserConfigFile, SuperUserCreateInitialSuperUser, SuperUserConfigValidate) -from app import CONFIG_PROVIDER +from app import config_provider from data.database import User import unittest @@ -10,11 +10,11 @@ import unittest class ConfigForTesting(object): def __enter__(self): - CONFIG_PROVIDER.reset_for_test() - return CONFIG_PROVIDER + config_provider.reset_for_test() + return config_provider def __exit__(self, type, value, traceback): - CONFIG_PROVIDER.reset_for_test() + config_provider.reset_for_test() class TestSuperUserRegistryStatus(ApiTestCase): @@ -166,7 +166,7 @@ class TestSuperUserConfig(ApiTestCase): self.assertTrue(json['exists']) # Verify the config file exists. - self.assertTrue(config.yaml_exists()) + self.assertTrue(config.config_exists()) # Try writing it again. This should now fail, since the config.yaml exists. self.putResponse(SuperUserConfig, data=dict(config={}, hostname='barbaz'), expected_code=403) diff --git a/util/config/provider.py b/util/config/provider.py deleted file mode 100644 index 32bc4c6f9..000000000 --- a/util/config/provider.py +++ /dev/null @@ -1,181 +0,0 @@ -import os -import yaml -import logging -import json -from StringIO import StringIO - -logger = logging.getLogger(__name__) - -class CannotWriteConfigException(Exception): - """ Exception raised when the config cannot be written. """ - pass - -def _import_yaml(config_obj, config_file): - with open(config_file) as f: - c = yaml.safe_load(f) - if not c: - logger.debug('Empty YAML config file') - return - - if isinstance(c, str): - raise Exception('Invalid YAML config file: ' + str(c)) - - for key in c.iterkeys(): - if key.isupper(): - config_obj[key] = c[key] - - return config_obj - - -def _export_yaml(config_obj, config_file): - try: - with open(config_file, 'w') as f: - f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)) - except IOError as ioe: - raise CannotWriteConfigException(str(ioe)) - - -class BaseProvider(object): - """ A configuration provider helps to load, save, and handle config override in the application. - """ - - def update_app_config(self, app_config): - """ Updates the given application config object with the loaded override config. """ - raise NotImplementedError - - def get_yaml(self): - """ Returns the contents of the YAML config override file, or None if none. """ - raise NotImplementedError - - def save_yaml(self, config_object): - """ Updates the contents of the YAML config override file to those given. """ - raise NotImplementedError - - def yaml_exists(self): - """ Returns true if a YAML config override file exists in the config volume. """ - raise NotImplementedError - - def volume_exists(self): - """ Returns whether the config override volume exists. """ - raise NotImplementedError - - def volume_file_exists(self, filename): - """ Returns whether the file with the given name exists under the config override volume. """ - raise NotImplementedError - - def get_volume_file(self, filename, mode='r'): - """ Returns a Python file referring to the given name under the config override volumne. """ - raise NotImplementedError - - def save_volume_file(self, filename, flask_file): - """ Saves the given flask file to the config override volume, with the given - filename. - """ - raise NotImplementedError - - def requires_restart(self, app_config): - """ If true, the configuration loaded into memory for the app does not match that on disk, - indicating that this container requires a restart. - """ - raise NotImplementedError - - -class FileConfigProvider(BaseProvider): - """ Implementation of the config provider that reads the data from the file system. """ - def __init__(self, config_volume, yaml_filename, py_filename): - self.config_volume = config_volume - self.yaml_filename = yaml_filename - self.py_filename = py_filename - - self.yaml_path = os.path.join(config_volume, yaml_filename) - self.py_path = os.path.join(config_volume, py_filename) - - def update_app_config(self, app_config): - if os.path.exists(self.py_path): - logger.debug('Applying config file: %s', self.py_path) - app_config.from_pyfile(self.py_path) - - if os.path.exists(self.yaml_path): - logger.debug('Applying config file: %s', self.yaml_path) - _import_yaml(app_config, self.yaml_path) - - def get_yaml(self): - if not os.path.exists(self.yaml_path): - return None - - config_obj = {} - _import_yaml(config_obj, self.yaml_path) - return config_obj - - def save_yaml(self, config_obj): - _export_yaml(config_obj, self.yaml_path) - - def yaml_exists(self): - return self.volume_file_exists(self.yaml_filename) - - def volume_exists(self): - return os.path.exists(self.config_volume) - - def volume_file_exists(self, filename): - return os.path.exists(os.path.join(self.config_volume, filename)) - - def get_volume_file(self, filename, mode='r'): - return open(os.path.join(self.config_volume, filename), mode) - - def save_volume_file(self, filename, flask_file): - try: - flask_file.save(os.path.join(self.config_volume, filename)) - except IOError as ioe: - raise CannotWriteConfigException(str(ioe)) - - def requires_restart(self, app_config): - file_config = self.get_yaml() - if not file_config: - return False - - for key in file_config: - if app_config.get(key) != file_config[key]: - return True - - return False - -class TestConfigProvider(BaseProvider): - """ Implementation of the config provider for testing. Everything is kept in-memory instead on - the real file system. """ - def __init__(self): - self.files = {} - self._config = None - - def update_app_config(self, app_config): - self._config = app_config - - def get_yaml(self): - if not 'config.yaml' in self.files: - return None - - return json.loads(self.files.get('config.yaml', '{}')) - - def save_yaml(self, config_obj): - self.files['config.yaml'] = json.dumps(config_obj) - - def yaml_exists(self): - return 'config.yaml' in self.files - - def volume_exists(self): - return True - - def volume_file_exists(self, filename): - return filename in self.files - - def save_volume_file(self, filename, flask_file): - self.files[filename] = '' - - def get_volume_file(self, filename, mode='r'): - return StringIO(self.files[filename]) - - def requires_restart(self, app_config): - return False - - def reset_for_test(self): - self._config['SUPER_USERS'] = ['devtable'] - self.files = {} diff --git a/util/config/provider/__init__.py b/util/config/provider/__init__.py new file mode 100644 index 000000000..ce2301912 --- /dev/null +++ b/util/config/provider/__init__.py @@ -0,0 +1,16 @@ +from util.config.provider.fileprovider import FileConfigProvider +from util.config.provider.testprovider import TestConfigProvider +from util.config.provider.k8sprovider import KubernetesConfigProvider + +import os + +def get_config_provider(config_volume, yaml_filename, py_filename, testing=False, kubernetes=False): + """ Loads and returns the config provider for the current environment. """ + if testing: + return TestConfigProvider() + + if kubernetes: + return KubernetesConfigProvider(config_volume, yaml_filename, py_filename) + + return FileConfigProvider(config_volume, yaml_filename, py_filename) + diff --git a/util/config/provider/baseprovider.py b/util/config/provider/baseprovider.py new file mode 100644 index 000000000..1cf9c654f --- /dev/null +++ b/util/config/provider/baseprovider.py @@ -0,0 +1,84 @@ +import yaml +import logging + +logger = logging.getLogger(__name__) + +class CannotWriteConfigException(Exception): + """ Exception raised when the config cannot be written. """ + pass + +def import_yaml(config_obj, config_file): + with open(config_file) as f: + c = yaml.safe_load(f) + if not c: + logger.debug('Empty YAML config file') + return + + if isinstance(c, str): + raise Exception('Invalid YAML config file: ' + str(c)) + + for key in c.iterkeys(): + if key.isupper(): + config_obj[key] = c[key] + + return config_obj + + +def get_yaml(config_obj): + return yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True) + +def export_yaml(config_obj, config_file): + try: + with open(config_file, 'w') as f: + f.write(get_yaml(config_obj)) + except IOError as ioe: + raise CannotWriteConfigException(str(ioe)) + + +class BaseProvider(object): + """ A configuration provider helps to load, save, and handle config override in the application. + """ + + @property + def provider_id(self): + raise NotImplementedError + + def update_app_config(self, app_config): + """ Updates the given application config object with the loaded override config. """ + raise NotImplementedError + + def get_config(self): + """ Returns the contents of the config override file, or None if none. """ + raise NotImplementedError + + def save_config(self, config_object): + """ Updates the contents of the config override file to those given. """ + raise NotImplementedError + + def config_exists(self): + """ Returns true if a config override file exists in the config volume. """ + raise NotImplementedError + + def volume_exists(self): + """ Returns whether the config override volume exists. """ + raise NotImplementedError + + def volume_file_exists(self, filename): + """ Returns whether the file with the given name exists under the config override volume. """ + raise NotImplementedError + + def get_volume_file(self, filename, mode='r'): + """ Returns a Python file referring to the given name under the config override volumne. """ + raise NotImplementedError + + def save_volume_file(self, filename, flask_file): + """ Saves the given flask file to the config override volume, with the given + filename. + """ + raise NotImplementedError + + def requires_restart(self, app_config): + """ If true, the configuration loaded into memory for the app does not match that on disk, + indicating that this container requires a restart. + """ + raise NotImplementedError \ No newline at end of file diff --git a/util/config/provider/fileprovider.py b/util/config/provider/fileprovider.py new file mode 100644 index 000000000..fe2b69df5 --- /dev/null +++ b/util/config/provider/fileprovider.py @@ -0,0 +1,73 @@ +import os +import logging + +from util.config.provider.baseprovider import (BaseProvider, import_yaml, export_yaml, + CannotWriteConfigException) + +logger = logging.getLogger(__name__) + +class FileConfigProvider(BaseProvider): + """ Implementation of the config provider that reads the data from the file system. """ + def __init__(self, config_volume, yaml_filename, py_filename): + self.config_volume = config_volume + self.yaml_filename = yaml_filename + self.py_filename = py_filename + + self.yaml_path = os.path.join(config_volume, yaml_filename) + self.py_path = os.path.join(config_volume, py_filename) + + @property + def provider_id(self): + return 'file' + + def update_app_config(self, app_config): + if os.path.exists(self.py_path): + logger.debug('Applying config file: %s', self.py_path) + app_config.from_pyfile(self.py_path) + + if os.path.exists(self.yaml_path): + logger.debug('Applying config file: %s', self.yaml_path) + import_yaml(app_config, self.yaml_path) + + def get_config(self): + if not os.path.exists(self.yaml_path): + return None + + config_obj = {} + import_yaml(config_obj, self.yaml_path) + return config_obj + + def save_config(self, config_obj): + export_yaml(config_obj, self.yaml_path) + + def config_exists(self): + return self.volume_file_exists(self.yaml_filename) + + def volume_exists(self): + return os.path.exists(self.config_volume) + + def volume_file_exists(self, filename): + return os.path.exists(os.path.join(self.config_volume, filename)) + + def get_volume_file(self, filename, mode='r'): + return open(os.path.join(self.config_volume, filename), mode) + + def save_volume_file(self, filename, flask_file): + filepath = os.path.join(self.config_volume, filename) + try: + flask_file.save(filepath) + except IOError as ioe: + raise CannotWriteConfigException(str(ioe)) + + return filepath + + def requires_restart(self, app_config): + file_config = self.get_config() + if not file_config: + return False + + for key in file_config: + if app_config.get(key) != file_config[key]: + return True + + return False diff --git a/util/config/provider/k8sprovider.py b/util/config/provider/k8sprovider.py new file mode 100644 index 000000000..4a5a6ef9d --- /dev/null +++ b/util/config/provider/k8sprovider.py @@ -0,0 +1,109 @@ +import os +import logging +import json +import base64 + +from requests import Request, Session + +from util.config.provider.baseprovider import get_yaml, CannotWriteConfigException +from util.config.provider.fileprovider import FileConfigProvider + +logger = logging.getLogger(__name__) + +KUBERNETES_API_HOST = 'kubernetes.default.svc.cluster.local' + +SERVICE_ACCOUNT_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token' + +ER_NAMESPACE = 'quay' +ER_CONFIG_SECRET = 'quay-config-secret' + +class KubernetesConfigProvider(FileConfigProvider): + """ Implementation of the config provider that reads and writes configuration + data from a Kubernetes Secret. """ + def __init__(self, config_volume, yaml_filename, py_filename): + super(KubernetesConfigProvider, self).__init__(config_volume, yaml_filename, py_filename) + + self.yaml_filename = yaml_filename + + # Load the service account token from the local store. + if not os.path.exists(SERVICE_ACCOUNT_TOKEN_PATH): + raise Exception('Cannot load Kubernetes service account token') + + with open(SERVICE_ACCOUNT_TOKEN_PATH, 'r') as f: + self._service_token = f.read() + + # Make sure the configuration volume exists. + if not self.volume_exists(): + os.makedirs(config_volume) + + @property + def provider_id(self): + return 'k8s' + + def save_config(self, config_obj): + self._update_secret_file(self.yaml_filename, get_yaml(config_obj)) + super(KubernetesConfigProvider, self).save_config(config_obj) + + def save_volume_file(self, filename, flask_file): + filepath = super(KubernetesConfigProvider, self).save_volume_file(filename, flask_file) + + try: + with open(filepath, 'r') as f: + self._update_secret_file(filename, f.read()) + except IOError as ioe: + raise CannotWriteConfigException(str(ioe)) + + def _assert_success(self, response): + if response.status_code != 200: + logger.error('K8s API call failed with response: %s => %s', response.status_code, + response.text) + raise CannotWriteConfigException('K8s API call failed. Please report this to support') + + def _update_secret_file(self, filename, value): + secret_data = {} + secret_data[filename] = base64.b64encode(value) + + data = { + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": ER_CONFIG_SECRET + }, + "data": secret_data + } + + secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET) + secret = self._lookup_secret() + if not secret: + self._assert_success(self._execute_k8s_api('POST', secret_url, data)) + return + + if not 'data' in secret: + secret['data'] = {} + + secret['data'][filename] = base64.b64encode(value) + self._assert_success(self._execute_k8s_api('PUT', secret_url, secret)) + + + def _lookup_secret(self): + secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET) + response = self._execute_k8s_api('GET', secret_url) + if response.status_code != 200: + return None + + return json.loads(response.text) + + def _execute_k8s_api(self, method, relative_url, data=None): + headers = { + 'Authorization': 'Bearer ' + self._service_token + } + + if data: + headers['Content-Type'] = 'application/json' + + data = json.dumps(data) if data else None + session = Session() + url = 'https://%s/api/v1/%s' % (KUBERNETES_API_HOST, relative_url) + + request = Request(method, url, data=data, headers=headers) + return session.send(request.prepare(), verify=False, timeout=2) diff --git a/util/config/provider/testprovider.py b/util/config/provider/testprovider.py new file mode 100644 index 000000000..96ccf77ae --- /dev/null +++ b/util/config/provider/testprovider.py @@ -0,0 +1,49 @@ +import json +from StringIO import StringIO + +from util.config.provider.baseprovider import BaseProvider + +class TestConfigProvider(BaseProvider): + """ Implementation of the config provider for testing. Everything is kept in-memory instead on + the real file system. """ + def __init__(self): + self.files = {} + self._config = None + + @property + def provider_id(self): + return 'test' + + def update_app_config(self, app_config): + self._config = app_config + + def get_config(self): + if not 'config.yaml' in self.files: + return None + + return json.loads(self.files.get('config.yaml', '{}')) + + def save_config(self, config_obj): + self.files['config.yaml'] = json.dumps(config_obj) + + def config_exists(self): + return 'config.yaml' in self.files + + def volume_exists(self): + return True + + def volume_file_exists(self, filename): + return filename in self.files + + def save_volume_file(self, filename, flask_file): + self.files[filename] = '' + + def get_volume_file(self, filename, mode='r'): + return StringIO(self.files[filename]) + + def requires_restart(self, app_config): + return False + + def reset_for_test(self): + self._config['SUPER_USERS'] = ['devtable'] + self.files = {} diff --git a/util/config/validator.py b/util/config/validator.py index 67def6a76..e66cb341d 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -19,7 +19,7 @@ from auth.auth_context import get_authenticated_user from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig from bitbucket import BitBucket -from app import app, CONFIG_PROVIDER, get_app_url, OVERRIDE_CONFIG_DIRECTORY +from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY logger = logging.getLogger(__name__) @@ -223,10 +223,10 @@ def _validate_ssl(config, _): return for filename in SSL_FILENAMES: - if not CONFIG_PROVIDER.volume_file_exists(filename): + if not config_provider.volume_file_exists(filename): raise Exception('Missing required SSL file: %s' % filename) - with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[0]) as f: + with config_provider.get_volume_file(SSL_FILENAMES[0]) as f: cert_contents = f.read() # Validate the certificate. @@ -239,7 +239,7 @@ def _validate_ssl(config, _): raise Exception('The specified SSL certificate has expired.') private_key_path = None - with CONFIG_PROVIDER.get_volume_file(SSL_FILENAMES[1]) as f: + with config_provider.get_volume_file(SSL_FILENAMES[1]) as f: private_key_path = f.name if not private_key_path: