Add Kubernetes configuration provider which writes config to a secret

Fixes #145

app.py

@@ -32,7 +32,6 @@ from util.config.oauth import (GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuth
 from util.security.signing import Signer
 from util.saas.cloudwatch import start_cloudwatch_sender
 from util.saas.metricqueue import MetricQueue
-from util.saas.queuemetrics import QueueMetrics
 from util.config.provider import get_config_provider
 from util.config.configutil import generate_secret_key
 from util.config.superusermanager import SuperUserManager
@@ -55,10 +54,11 @@ class RegexConverter(BaseConverter):
 
 app.url_map.converters['regex'] = RegexConverter
 
-# Instantiate the default configuration (for test or for normal operation).
+# Instantiate the configuration.
 is_testing = 'TEST' in os.environ
+is_kubernetes = 'KUBERNETES_SERVICE_HOST' in os.environ
 config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py',
-                                      testing=is_testing)
+                                      testing=is_testing, kubernetes=is_kubernetes)
 
 if is_testing:
   from test.testconfig import TestConfig

endpoints/api/suconfig.py

@@ -62,7 +62,7 @@ class SuperUserRegistryStatus(ApiResource):
       }
 
     # If there is no config file, we need to setup the database.
-    if not config_provider.yaml_exists():
+    if not config_provider.config_exists():
       return {
         'status': 'config-db'
       }
@@ -107,10 +107,10 @@ class SuperUserSetupDatabase(ApiResource):
     """ Invokes the alembic upgrade process. """
     # Note: This method is called after the database configured is saved, but before the
     # database has any tables. Therefore, we only allow it to be run in that unique case.
-    if config_provider.yaml_exists() and not database_is_valid():
+    if config_provider.config_exists() and not database_is_valid():
       # Note: We need to reconfigure the database here as the config has changed.
       combined = dict(**app.config)
-      combined.update(config_provider.get_yaml())
+      combined.update(config_provider.get_config())
 
       configure(combined)
       app.config['DB_URI'] = combined['DB_URI']
@@ -185,7 +185,7 @@ class SuperUserConfig(ApiResource):
   def get(self):
     """ Returns the currently defined configuration, if any. """
     if SuperUserPermission().can():
-      config_object = config_provider.get_yaml()
+      config_object = config_provider.get_config()
       return {
         'config': config_object
       }
@@ -196,18 +196,18 @@ class SuperUserConfig(ApiResource):
   @verify_not_prod
   @validate_json_request('UpdateConfig')
   def put(self):
-    """ Updates the config.yaml file. """
+    """ Updates the config override file. """
     # Note: This method is called to set the database configuration before super users exists,
     # so we also allow it to be called if there is no valid registry configuration setup.
-    if not config_provider.yaml_exists() or SuperUserPermission().can():
+    if not config_provider.config_exists() or SuperUserPermission().can():
       config_object = request.get_json()['config']
       hostname = request.get_json()['hostname']
 
       # Add any enterprise defaults missing from the config.
       add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname)
 
-      # Write the configuration changes to the YAML file.
-      config_provider.save_yaml(config_object)
+      # Write the configuration changes to the config override file.
+      config_provider.save_config(config_object)
 
       # If the authentication system is not the database, link the superuser account to the
       # the authentication system chosen.
@@ -252,7 +252,7 @@ class SuperUserConfigFile(ApiResource):
 
     # Note: This method can be called before the configuration exists
     # to upload the database SSL cert.
-    if not config_provider.yaml_exists() or SuperUserPermission().can():
+    if not config_provider.config_exists() or SuperUserPermission().can():
       uploaded_file = request.files['file']
       if not uploaded_file:
         abort(400)
@@ -309,7 +309,7 @@ class SuperUserCreateInitialSuperUser(ApiResource):
     #
     # We do this special security check because at the point this method is called, the database
     # is clean but does not (yet) have any super users for our permissions code to check against.
-    if config_provider.yaml_exists() and not database_has_users():
+    if config_provider.config_exists() and not database_has_users():
       data = request.get_json()
       username = data['username']
       password = data['password']
@@ -319,9 +319,9 @@ class SuperUserCreateInitialSuperUser(ApiResource):
       superuser = model.user.create_user(username, password, email, auto_verify=True)
 
       # Add the user to the config.
-      config_object = config_provider.get_yaml()
+      config_object = config_provider.get_config()
       config_object['SUPER_USERS'] = [username]
-      config_provider.save_yaml(config_object)
+      config_provider.save_config(config_object)
 
       # Update the in-memory config for the new superuser.
       superusers.register_superuser(username)
@@ -369,7 +369,7 @@ class SuperUserConfigValidate(ApiResource):
     # Note: This method is called to validate the database configuration before super users exists,
     # so we also allow it to be called if there is no valid registry configuration setup. Note that
     # this is also safe since this method does not access any information not given in the request.
-    if not config_provider.yaml_exists() or SuperUserPermission().can():
+    if not config_provider.config_exists() or SuperUserPermission().can():
       config = request.get_json()['config']
       return validate_service_for_config(service, config, request.get_json().get('password', ''))
 

endpoints/web.py

@@ -9,7 +9,7 @@ from health.healthcheck import get_healthchecker
 
 from data import model
 from data.database import db
-from app import app, billing as stripe, build_logs, avatar, signer, log_archive
+from app import app, billing as stripe, build_logs, avatar, signer, log_archive, config_provider
 from auth.auth import require_session_login, process_oauth
 from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission,
                               SuperUserPermission, AdministerRepositoryPermission,
@@ -209,7 +209,7 @@ def v1():
 @web.route('/health/instance', methods=['GET'])
 @no_cache
 def instance_health():
-  checker = get_healthchecker(app)
+  checker = get_healthchecker(app, config_provider)
   (data, status_code) = checker.check_instance()
   response = jsonify(dict(data=data, status_code=status_code))
   response.status_code = status_code
@@ -221,7 +221,7 @@ def instance_health():
 @web.route('/health/endtoend', methods=['GET'])
 @no_cache
 def endtoend_health():
-  checker = get_healthchecker(app)
+  checker = get_healthchecker(app, config_provider)
   (data, status_code) = checker.check_endtoend()
   response = jsonify(dict(data=data, status_code=status_code))
   response.status_code = status_code

health/healthcheck.py

@@ -4,14 +4,15 @@ from health.services import check_all_services
 
 logger = logging.getLogger(__name__)
 
-def get_healthchecker(app):
+def get_healthchecker(app, config_provider):
   """ Returns a HealthCheck instance for the given app. """
-  return HealthCheck.get_checker(app)
+  return HealthCheck.get_checker(app, config_provider)
 
 
 class HealthCheck(object):
-  def __init__(self, app):
+  def __init__(self, app, config_provider):
     self.app = app
+    self.config_provider = config_provider
 
   def check_instance(self):
     """
@@ -52,20 +53,21 @@ class HealthCheck(object):
     data = {
       'services': service_statuses,
       'notes': notes,
-      'is_testing': self.app.config['TESTING']
+      'is_testing': self.app.config['TESTING'],
+      'config_provider': self.config_provider.provider_id
     }
 
     return (data, 200 if is_healthy else 503)
 
 
   @classmethod
-  def get_checker(cls, app):
+  def get_checker(cls, app, config_provider):
     name = app.config['HEALTH_CHECKER'][0]
     parameters = app.config['HEALTH_CHECKER'][1] or {}
 
     for subc in cls.__subclasses__():
       if subc.check_name() == name:
-        return subc(app, **parameters)
+        return subc(app, config_provider, **parameters)
 
     raise Exception('Unknown health check with name %s' % name)
 
@@ -77,8 +79,8 @@ class LocalHealthCheck(HealthCheck):
 
 
 class ProductionHealthCheck(HealthCheck):
-  def __init__(self, app, access_key, secret_key, db_instance='quay'):
-    super(ProductionHealthCheck, self).__init__(app)
+  def __init__(self, app, config_provider, access_key, secret_key, db_instance='quay'):
+    super(ProductionHealthCheck, self).__init__(app, config_provider)
     self.access_key = access_key
     self.secret_key = secret_key
     self.db_instance = db_instance

test/test_suconfig_api.py

@@ -166,7 +166,7 @@ class TestSuperUserConfig(ApiTestCase):
       self.assertTrue(json['exists'])
 
       # Verify the config file exists.
-      self.assertTrue(config.yaml_exists())
+      self.assertTrue(config.config_exists())
 
       # Try writing it again. This should now fail, since the config.yaml exists.
       self.putResponse(SuperUserConfig, data=dict(config={}, hostname='barbaz'), expected_code=403)

util/config/provider/__init__.py

@@ -1,10 +1,16 @@
 from util.config.provider.fileprovider import FileConfigProvider
 from util.config.provider.testprovider import TestConfigProvider
+from util.config.provider.k8sprovider import KubernetesConfigProvider
 
-def get_config_provider(config_volume, yaml_filename, py_filename, testing=False):
+import os
+
+def get_config_provider(config_volume, yaml_filename, py_filename, testing=False, kubernetes=False):
   """ Loads and returns the config provider for the current environment. """
   if testing:
     return TestConfigProvider()
 
+  if kubernetes:
+    return KubernetesConfigProvider(config_volume, yaml_filename, py_filename)
+
   return FileConfigProvider(config_volume, yaml_filename, py_filename)
 

util/config/provider/baseprovider.py

@@ -24,10 +24,13 @@ def import_yaml(config_obj, config_file):
   return config_obj
 
 
+def get_yaml(config_obj):
+  return yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)
+
 def export_yaml(config_obj, config_file):
   try:
     with open(config_file, 'w') as f:
-      f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True))
+      f.write(get_yaml(config_obj))
   except IOError as ioe:
     raise CannotWriteConfigException(str(ioe))
 
@@ -36,20 +39,24 @@ class BaseProvider(object):
   """ A configuration provider helps to load, save, and handle config override in the application.
   """
 
+  @property
+  def provider_id(self):
+    raise NotImplementedError
+
   def update_app_config(self, app_config):
     """ Updates the given application config object with the loaded override config. """
     raise NotImplementedError
 
-  def get_yaml(self):
-    """ Returns the contents of the YAML config override file, or None if none. """
+  def get_config(self):
+    """ Returns the contents of the config override file, or None if none. """
     raise NotImplementedError
 
-  def save_yaml(self, config_object):
-    """ Updates the contents of the YAML config override file to those given. """
+  def save_config(self, config_object):
+    """ Updates the contents of the config override file to those given. """
     raise NotImplementedError
 
-  def yaml_exists(self):
-    """ Returns true if a YAML config override file exists in the config volume. """
+  def config_exists(self):
+    """ Returns true if a config override file exists in the config volume. """
     raise NotImplementedError
 
   def volume_exists(self):

util/config/provider/fileprovider.py

@@ -16,6 +16,10 @@ class FileConfigProvider(BaseProvider):
     self.yaml_path = os.path.join(config_volume, yaml_filename)
     self.py_path = os.path.join(config_volume, py_filename)
 
+  @property
+  def provider_id(self):
+    return 'file'
+
   def update_app_config(self, app_config):
     if os.path.exists(self.py_path):
       logger.debug('Applying config file: %s', self.py_path)
@@ -25,7 +29,7 @@ class FileConfigProvider(BaseProvider):
       logger.debug('Applying config file: %s', self.yaml_path)
       import_yaml(app_config, self.yaml_path)
 
-  def get_yaml(self):
+  def get_config(self):
     if not os.path.exists(self.yaml_path):
       return None
 
@@ -33,10 +37,10 @@ class FileConfigProvider(BaseProvider):
     import_yaml(config_obj, self.yaml_path)
     return config_obj
 
-  def save_yaml(self, config_obj):
+  def save_config(self, config_obj):
     export_yaml(config_obj, self.yaml_path)
 
-  def yaml_exists(self):
+  def config_exists(self):
     return self.volume_file_exists(self.yaml_filename)
 
   def volume_exists(self):
@@ -49,13 +53,16 @@ class FileConfigProvider(BaseProvider):
     return open(os.path.join(self.config_volume, filename), mode)
 
   def save_volume_file(self, filename, flask_file):
+    filepath = os.path.join(self.config_volume, filename)
     try:
-      flask_file.save(os.path.join(self.config_volume, filename))
+      flask_file.save(filepath)
     except IOError as ioe:
       raise CannotWriteConfigException(str(ioe))
 
+    return filepath
+
   def requires_restart(self, app_config):
-    file_config = self.get_yaml()
+    file_config = self.get_config()
     if not file_config:
       return False
 

util/config/provider/k8sprovider.py

@@ -0,0 +1,109 @@
+import os
+import logging
+import json
+import base64
+
+from requests import Request, Session
+
+from util.config.provider.baseprovider import get_yaml, CannotWriteConfigException
+from util.config.provider.fileprovider import FileConfigProvider
+
+logger = logging.getLogger(__name__)
+
+KUBERNETES_API_HOST = 'kubernetes.default.svc.cluster.local'
+
+SERVICE_ACCOUNT_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token'
+
+ER_NAMESPACE = 'quay'
+ER_CONFIG_SECRET = 'quay-config-secret'
+
+class KubernetesConfigProvider(FileConfigProvider):
+  """ Implementation of the config provider that reads and writes configuration
+      data from a Kubernetes Secret. """
+  def __init__(self, config_volume, yaml_filename, py_filename):
+    super(KubernetesConfigProvider, self).__init__(config_volume, yaml_filename, py_filename)
+
+    self.yaml_filename = yaml_filename
+
+    # Load the service account token from the local store.
+    if not os.path.exists(SERVICE_ACCOUNT_TOKEN_PATH):
+      raise Exception('Cannot load Kubernetes service account token')
+
+    with open(SERVICE_ACCOUNT_TOKEN_PATH, 'r') as f:
+      self._service_token = f.read()
+
+    # Make sure the configuration volume exists.
+    if not self.volume_exists():
+      os.makedirs(config_volume)
+
+  @property
+  def provider_id(self):
+    return 'k8s'
+
+  def save_config(self, config_obj):
+    self._update_secret_file(self.yaml_filename, get_yaml(config_obj))
+    super(KubernetesConfigProvider, self).save_config(config_obj)
+
+  def save_volume_file(self, filename, flask_file):
+    filepath = super(KubernetesConfigProvider, self).save_volume_file(filename, flask_file)
+
+    try:
+      with open(filepath, 'r') as f:
+        self._update_secret_file(filename, f.read())
+    except IOError as ioe:
+      raise CannotWriteConfigException(str(ioe))
+
+  def _assert_success(self, response):
+    if response.status_code != 200:
+      logger.error('K8s API call failed with response: %s => %s', response.status_code,
+                   response.text)
+      raise CannotWriteConfigException('K8s API call failed. Please report this to support')
+
+  def _update_secret_file(self, filename, value):
+    secret_data = {}
+    secret_data[filename] = base64.b64encode(value)
+
+    data = {
+      "kind": "Secret",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": ER_CONFIG_SECRET
+      },
+      "data": secret_data
+    }
+
+    secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET)
+    secret = self._lookup_secret()
+    if not secret:
+      self._assert_success(self._execute_k8s_api('POST', secret_url, data))
+      return
+
+    if not 'data' in secret:
+      secret['data'] = {}
+
+    secret['data'][filename] = base64.b64encode(value)
+    self._assert_success(self._execute_k8s_api('PUT', secret_url, secret))
+
+
+  def _lookup_secret(self):
+    secret_url = 'namespaces/%s/secrets/%s' % (ER_NAMESPACE, ER_CONFIG_SECRET)
+    response = self._execute_k8s_api('GET', secret_url)
+    if response.status_code != 200:
+      return None
+
+    return json.loads(response.text)
+
+  def _execute_k8s_api(self, method, relative_url, data=None):
+    headers = {
+      'Authorization': 'Bearer ' + self._service_token
+    }
+
+    if data:
+      headers['Content-Type'] = 'application/json'
+
+    data = json.dumps(data) if data else None
+    session = Session()
+    url = 'https://%s/api/v1/%s' % (KUBERNETES_API_HOST, relative_url)
+
+    request = Request(method, url, data=data, headers=headers)
+    return session.send(request.prepare(), verify=False, timeout=2)

util/config/provider/testprovider.py

@@ -10,19 +10,23 @@ class TestConfigProvider(BaseProvider):
     self.files = {}
     self._config = None
 
+  @property
+  def provider_id(self):
+    return 'test'
+
   def update_app_config(self, app_config):
     self._config = app_config
 
-  def get_yaml(self):
+  def get_config(self):
     if not 'config.yaml' in self.files:
       return None
 
     return json.loads(self.files.get('config.yaml', '{}'))
 
-  def save_yaml(self, config_obj):
+  def save_config(self, config_obj):
     self.files['config.yaml'] = json.dumps(config_obj)
 
-  def yaml_exists(self):
+  def config_exists(self):
     return 'config.yaml' in self.files
 
   def volume_exists(self):