Get end-to-end configuration setup working, including verification (except for Github, which is in progress)

util/config/configutil.py

@@ -0,0 +1,77 @@
+import yaml
+
+from random import SystemRandom
+
+def generate_secret_key():
+  cryptogen = SystemRandom()
+  return  str(cryptogen.getrandbits(256))
+
+
+def import_yaml(config_obj, config_file):
+  with open(config_file) as f:
+    c = yaml.safe_load(f)
+    if not c:
+      logger.debug('Empty YAML config file')
+      return
+
+    if isinstance(c, str):
+      raise Exception('Invalid YAML config file: ' + str(c))
+
+    for key in c.iterkeys():
+      if key.isupper():
+        config_obj[key] = c[key]
+
+
+def export_yaml(config_obj, config_file):
+  with open(config_file, 'w') as f:
+    f.write(yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True))
+
+
+def set_config_value(config_file, config_key, value):
+  """ Loads the configuration from the given YAML config file, sets the given key to
+      the given value, and then writes it back out to the given YAML config file. """
+  config_obj = {}
+  import_yaml(config_obj, config_file)
+  config_obj[config_key] = value
+  export_yaml(config_obj, config_file)
+
+
+def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
+  """ Adds/Sets the config defaults for enterprise registry config. """
+  # These have to be false.
+  config_obj['TESTING'] = False
+  config_obj['USE_CDN'] = False
+
+  # Default features that are on.
+  config_obj['FEATURE_USER_LOG_ACCESS'] = config_obj.get('FEATURE_USER_LOG_ACCESS', True)
+  config_obj['FEATURE_USER_CREATION'] = config_obj.get('FEATURE_USER_CREATION', True)
+
+  # Default features that are off.
+  config_obj['FEATURE_MAILING'] = config_obj.get('FEATURE_MAILING', False)
+  config_obj['FEATURE_BUILD_SUPPORT'] = config_obj.get('FEATURE_BUILD_SUPPORT', False)
+
+  # Default auth type.
+  if not 'AUTHENTICATION_TYPE' in config_obj:
+    config_obj['AUTHENTICATION_TYPE'] = 'Database'
+
+  # Default secret key.
+  if not 'SECRET_KEY' in config_obj:
+    config_obj['SECRET_KEY'] = current_secret_key
+
+  # Default storage configuration.
+  if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj:
+    config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['local']
+    config_obj['DISTRIBUTED_STORAGE_CONFIG'] = {
+      'local': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
+    }
+
+    config_obj['USERFILES_LOCATION'] = 'local'
+    config_obj['USERFILES_PATH'] = 'userfiles/'
+
+  if not 'SERVER_HOSTNAME' in config_obj:
+    config_obj['SERVER_HOSTNAME'] = hostname
+
+  # Misc configuration.
+  config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http')
+  config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get('ENTERPRISE_LOGO_URL',
+                                                     '/static/img/quay-logo.png')

util/config/validator.py

@@ -0,0 +1,122 @@
+import redis
+import os
+import json
+import ldap
+
+from data.users import LDAPConnection
+from flask import Flask
+from flask.ext.mail import Mail, Message
+from data.database import validate_database_url, User
+from storage import get_storage_driver
+from app import app, OVERRIDE_CONFIG_DIRECTORY
+from auth.auth_context import get_authenticated_user
+from util.oauth import GoogleOAuthConfig, GithubOAuthConfig
+
+SSL_FILENAMES = ['ssl.cert', 'ssl.key']
+
+def validate_service_for_config(service, config):
+  """ Attempts to validate the configuration for the given service. """
+  if not service in _VALIDATORS:
+    return {
+      'status': False
+    }
+
+  try:
+    _VALIDATORS[service](config)
+    return {
+      'status': True
+    }
+  except Exception as ex:
+    return {
+      'status': False,
+      'reason': str(ex)
+    }
+
+def _validate_database(config):
+  """ Validates connecting to the database. """
+  validate_database_url(config['DB_URI'])
+
+def _validate_redis(config):
+  """ Validates connecting to redis. """
+  redis_config = config['BUILDLOGS_REDIS']
+  client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
+  client.ping()
+
+def _validate_registry_storage(config):
+  """ Validates registry storage. """
+  parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}])
+  try:
+    driver = get_storage_driver(parameters)
+  except TypeError:
+    raise Exception('Missing required storage configuration parameter(s)')
+
+  # Put and remove a temporary file.
+  driver.put_content('_verify', 'testing 123')
+  driver.remove('_verify')
+
+def _validate_mailing(config):
+  """ Validates sending email. """
+  test_app = Flask("mail-test-app")
+  test_app.config.update(config)
+  test_app.config.update({
+    'MAIL_FAIL_SILENTLY': False,
+    'TESTING': False
+  })
+
+  test_mail = Mail(test_app)
+  test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'])
+  test_msg.add_recipient(get_authenticated_user().email)
+  test_mail.send(test_msg)
+
+def _validate_github_login(config):
+  """ Validates the OAuth credentials and API endpoint for Github Login. """
+  client = app.config['HTTPCLIENT']
+  oauth = GithubOAuthConfig(config, 'GITHUB_LOGIN_CONFIG')
+  endpoint = oauth.authorize_endpoint()
+  # TODO: this
+
+
+def _validate_ssl(config):
+  """ Validates the SSL configuration (if enabled). """
+  if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
+    return
+
+  for filename in SSL_FILENAMES:
+    if not os.path.exists(os.path.join(OVERRIDE_CONFIG_DIRECTORY, filename)):
+      raise Exception('Missing required SSL file: %s' % filename)
+
+
+def _validate_ldap(config):
+  """ Validates the LDAP connection. """
+  if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
+    return
+
+  # Note: raises ldap.INVALID_CREDENTIALS on failure
+  admin_dn = config.get('LDAP_ADMIN_DN')
+  admin_passwd = config.get('LDAP_ADMIN_PASSWD')
+
+  if not admin_dn:
+    raise Exception('Missing Admin DN for LDAP configuration')
+
+  if not admin_passwd:
+    raise Exception('Missing Admin Password for LDAP configuration')
+
+  ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
+
+  try:
+    with LDAPConnection(ldap_uri, admin_dn, admin_passwd):
+      pass
+  except ldap.LDAPError as ex:
+    values = ex.args[0] if ex.args else {}
+    raise Exception(values.get('desc', 'Unknown error'))
+
+
+_VALIDATORS = {
+  'database': _validate_database,
+  'redis': _validate_redis,
+  'registry-storage': _validate_registry_storage,
+  'mail': _validate_mailing,
+  'github-login': _validate_github_login,
+  'ssl': _validate_ssl,
+  'ldap': _validate_ldap,
+}