superuser: add storage replication config

util/config/configutil.py

@@ -32,15 +32,15 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
 
   # Default storage configuration.
   if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj:
-    config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['local']
+    config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['default']
     config_obj['DISTRIBUTED_STORAGE_CONFIG'] = {
-      'local': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
+      'default': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
     }
 
-    config_obj['USERFILES_LOCATION'] = 'local'
+    config_obj['USERFILES_LOCATION'] = 'default'
     config_obj['USERFILES_PATH'] = 'userfiles/'
 
-    config_obj['LOG_ARCHIVE_LOCATION'] = 'local'
+    config_obj['LOG_ARCHIVE_LOCATION'] = 'default'
 
   if not 'SERVER_HOSTNAME' in config_obj:
     config_obj['SERVER_HOSTNAME'] = hostname

util/config/database.py

@@ -0,0 +1,9 @@
+from data import model
+
+
+def sync_database_with_config(config):
+  """ This ensures all implicitly required reference table entries exist in the database. """
+
+  location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
+  if location_names:
+    model.image.ensure_image_locations(*location_names)

util/config/validator.py

@@ -30,12 +30,18 @@ JWT_FILENAMES = ['jwt-authn.cert']
 
 CONFIG_FILENAMES = SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES
 
-def get_storage_provider(config):
-  parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}])
+def get_storage_providers(config):
+  storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
+
+  drivers = {}
+
   try:
-    return get_storage_driver(parameters)
+    for name, parameters in storage_config.items():
+      drivers[name] = (parameters[0], get_storage_driver(parameters))
   except TypeError:
-    raise Exception('Missing required storage configuration parameter(s)')
+    raise Exception('Missing required storage configuration parameter(s): %s' % name)
+
+  return drivers
 
 def validate_service_for_config(service, config, password=None):
   """ Attempts to validate the configuration for the given service. """
@@ -80,20 +86,29 @@ def _validate_redis(config, _):
 
 def _validate_registry_storage(config, _):
   """ Validates registry storage. """
-  driver = get_storage_provider(config)
+  replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
 
-  # Run custom validation on the driver.
-  driver.validate(app.config['HTTPCLIENT'])
+  providers = get_storage_providers(config).items()
 
-  # Put and remove a temporary file to make sure the normal storage paths work.
-  driver.put_content('_verify', 'testing 123')
-  driver.remove('_verify')
+  if not providers:
+    raise Exception('Storage configuration required')
 
-  # Run setup on the driver if the read/write succeeded.
-  try:
-    driver.setup()
-  except Exception as ex:
-    raise Exception('Could not prepare storage: %s' % str(ex))
+  for name, (storage_type, driver) in providers:
+    try:
+      if replication_enabled and storage_type == 'LocalStorage':
+        raise Exception('Locally mounted directory not supported with storage replication')
+
+      # Run custom validation on the driver.
+      driver.validate(app.config['HTTPCLIENT'])
+
+      # Put and remove a temporary file to make sure the normal storage paths work.
+      driver.put_content('_verify', 'testing 123')
+      driver.remove('_verify')
+
+      # Run setup on the driver if the read/write succeeded.
+      driver.setup()
+    except Exception as ex:
+      raise Exception('Invalid storage configuration: %s: %s' % (name, str(ex)))
 
 
 def _validate_mailing(config, _):