superuser: add storage replication config

Dockerfile

@@ -43,7 +43,7 @@ ADD conf/init/doupdatelimits.sh /etc/my_init.d/
 ADD conf/init/copy_syslog_config.sh /etc/my_init.d/
 ADD conf/init/runmigration.sh /etc/my_init.d/
 ADD conf/init/syslog-ng.conf /etc/syslog-ng/
-ADD conf/init/zz_release.sh /etc/my_init.d/
+ADD conf/init/zz_boot.sh /etc/my_init.d/
 
 ADD conf/init/service/ /etc/service/
 

boot.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import release
+
+from app import app
+from data.model.release import set_region_release
+from util.config.database import sync_database_with_config
+
+
+def main():
+  if app.config.get('SETUP_COMPLETE', False):
+    sync_database_with_config(app.config)
+
+  # Record deploy
+  if release.REGION and release.GIT_HEAD:
+    set_region_release(release.SERVICE, release.REGION, release.GIT_HEAD)
+
+
+if __name__ == '__main__':
+  main()

conf/init/zz_boot.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+/venv/bin/python /boot.py

conf/init/zz_release.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -e
-
-source venv/bin/activate
-
-export PYTHONPATH=.
-
-python /release.py

data/model/image.py

@@ -356,3 +356,19 @@ def get_image(repo, dockerfile_id):
     return Image.get(Image.docker_image_id == dockerfile_id, Image.repository == repo)
   except Image.DoesNotExist:
     return None
+
+
+def ensure_image_locations(*names):
+  with db_transaction():
+    locations = ImageStorageLocation.select().where(ImageStorageLocation.name << names)
+
+    insert_names = list(names)
+
+    for location in locations:
+      insert_names.remove(location.name)
+
+    if not insert_names:
+      return
+
+    data = [{'name': name} for name in insert_names]
+    ImageStorageLocation.insert_many(data).execute()

endpoints/api/suconfig.py

@@ -16,6 +16,7 @@ from auth.permissions import SuperUserPermission
 from auth.auth_context import get_authenticated_user
 from data.database import User
 from util.config.configutil import add_enterprise_config_defaults
+from util.config.database import sync_database_with_config
 from util.config.validator import validate_service_for_config, CONFIG_FILENAMES
 from data.runmigration import run_alembic_migration
 from data.users import get_federated_service_name
@@ -216,6 +217,9 @@ class SuperUserConfig(ApiResource):
         current_user = get_authenticated_user()
         model.user.confirm_attached_federated_login(current_user, service_name)
 
+      # Ensure database is up-to-date with config
+      sync_database_with_config(config_object)
+
       return {
         'exists': True,
         'config': config_object

release.py

@@ -12,15 +12,3 @@ REGION = os.environ.get('QUAY_REGION')
 if os.path.isfile(_GIT_HEAD_PATH):
   with open(_GIT_HEAD_PATH) as f:
     GIT_HEAD = f.read().strip()
-
-
-def main():
-  from app import app
-  from data.model.release import set_region_release
-
-  if REGION and GIT_HEAD:
-    set_region_release(SERVICE, REGION, GIT_HEAD)
-
-
-if __name__ == '__main__':
-  main()

static/css/pages/superuser.css

@@ -23,3 +23,29 @@
 .super-user .user-row.disabled .avatar {
   -webkit-filter: grayscale(100%);
 }
+
+.super-user td .co-alert {
+  margin: 16px 0 0 0;
+}
+
+.super-user .add-storage-link {
+  margin-top: 5px;
+}
+
+.super-user .storage-config {
+  border-bottom: 1px solid #eee;
+  padding: 0 0 10px 0;
+  margin: 10px 0 0 0;
+}
+
+.super-user .last {
+  border-bottom: none;
+}
+
+.super-user .feature-storage-replication {
+  margin: 15px 0 10px 0;
+}
+
+.super-user .input-util {
+  margin-top: 10px;
+}

static/css/quay.css

@@ -1363,7 +1363,7 @@ i.toggle-icon:hover {
 
 form input.ng-invalid.ng-dirty,
 *[ng-form] input.ng-invalid.ng-dirty {
-  background-color: #FDD7D9;
+  background-color: #FDD7D9 !important;
 }
 
 form input.ng-valid.ng-dirty,

static/directives/config/config-setup-tool.html

@@ -194,54 +194,94 @@
             <strong>A remote storage system is required for high-avaliability systems.</strong>
           </p>
 
-          <table class="config-table">
-            <tr>
-              <td class="non-input">Storage Engine:</td>
-              <td>
-                <select ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[0]">
-                  <option value="LocalStorage">Locally mounted directory</option>
-                  <option value="S3Storage">Amazon S3</option>
-                  <option value="GoogleCloudStorage">Google Cloud Storage</option>
-                  <option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
-                  <option value="SwiftStorage">OpenStack Storage (Swift)</option>
-                </select>
-              </td>
-            </tr>
+          <div class="co-checkbox feature-storage-replication">
+            <input id="ftsr" type="checkbox" ng-model="config.FEATURE_STORAGE_REPLICATION">
+            <label for="ftsr">Enable Storage Replication</label>
+            <!-- TODO(josephschorr): add link to documentation -->
+            <div class="help-text">If enabled, replicates storage to other regions.</div>
+          </div>
 
-            <!-- Fields -->
-            <tr ng-repeat="field in STORAGE_CONFIG_FIELDS[config.DISTRIBUTED_STORAGE_CONFIG.local[0]]">
-              <td>{{ field.title }}:</td>
-              <td>
-                <span class="config-map-field"
-                      binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
-                      ng-if="field.kind == 'map'"
-                      keys="field.keys"></span>
-                <span class="config-string-field"
-                      binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
-                      placeholder="{{ field.placeholder }}"
-                      ng-if="field.kind == 'text'"
-                      is-optional="field.optional"></span>
-                <div class="co-checkbox" ng-if="field.kind == 'bool'">
-                  <input id="dsc-{{ field.name }}" type="checkbox"
-                         ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
-                  <label for="dsc-{{ field.name }}">{{ field.placeholder }}</label>
-                </div>
-                <div ng-if="field.kind == 'option'">
-                  <select ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
-                    <option ng-repeat="value in field.values" value="{{ value }}"
-                            ng-selected="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name] == value">{{ value }}</option>
+          <div class="storage-config" ng-class="$last ? 'last' : ''" ng-repeat="sc in storageConfig">
+            <table class="config-table">
+              <tr>
+                <td class="non-input">Location ID:</td>
+                <td>
+                  <input class="form-control" ng-if="allowChangeLocationStorageConfig(sc.location)" ng-class="storageConfigError[$index].location ? 'ng-invalid' : ''" ng-model="sc.location" ng-pattern="/^[a-zA-Z0-9_-]+$/" required>
+                  <div ng-if="!allowChangeLocationStorageConfig(sc.location)">
+                    {{ sc.location }}
+                  </div>
+                  <div class="co-alert co-alert-danger" ng-show="storageConfigError[$index].location">
+                    {{ storageConfigError[$index].location }}
+                  </div>
+                  <div class="input-util" ng-if="allowRemoveStorageConfig(sc.location)"><a href="javascript:void(0)" class="remove-link" ng-click="removeStorageConfig(sc)">Remove</a></div>
+                </td>
+              </tr>
+
+              <tr ng-if="config.FEATURE_STORAGE_REPLICATION">
+                <td class="non-input">Set Default:</td>
+                <td>
+                  <div class="co-checkbox">
+                    <input id="default-location-{{ $index }}" ng-model="sc.defaultLocation" type="checkbox">
+                    <label for="default-location-{{ $index }}">Replicate to storage engine by default</label>
+                  </div>
+                </td>
+              </tr>
+
+              <tr>
+                <td class="non-input">Storage Engine:</td>
+                <td>
+                  <select class="form-control" ng-class="storageConfigError[$index].engine ? 'ng-invalid' : ''" ng-model="sc.data[0]">
+                    <option value="LocalStorage">Locally mounted directory</option>
+                    <option value="S3Storage">Amazon S3</option>
+                    <option value="GoogleCloudStorage">Google Cloud Storage</option>
+                    <option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
+                    <option value="SwiftStorage">OpenStack Storage (Swift)</option>
                   </select>
-                </div>
-                <div class="help-text" ng-if="field.help_text">
-                  {{ field.help_text }}
-                </div>
-                <div class="help-text" ng-if="field.help_url">
-                  See <a href="{{ field.help_url }}" target="_blank">Documentation</a> for more information
-                </div>
-              </td>
-            </tr>
-          </table>
 
+                  <div class="co-alert co-alert-danger" ng-if="storageConfigError[$index].engine">
+                    {{ storageConfigError[$index].engine }}
+                  </div>
+                </td>
+              </tr>
+
+              <!-- Fields -->
+              <tr ng-repeat="field in STORAGE_CONFIG_FIELDS[sc.data[0]]">
+                <td>{{ field.title }}:</td>
+                <td>
+                  <span class="config-map-field"
+                        binding="sc.data[1][field.name]"
+                        ng-if="field.kind == 'map'"
+                        keys="field.keys"></span>
+                  <span class="config-string-field"
+                        binding="sc.data[1][field.name]"
+                        placeholder="{{ field.placeholder }}"
+                        ng-if="field.kind == 'text'"
+                        is-optional="field.optional"></span>
+                  <div class="co-checkbox" ng-if="field.kind == 'bool'">
+                    <input id="dsc-{{ field.name }}" type="checkbox"
+                           ng-model="sc.data[1][field.name]">
+                    <label for="dsc-{{ field.name }}">{{ field.placeholder }}</label>
+                  </div>
+                  <div ng-if="field.kind == 'option'">
+                    <select ng-model="sc.data[1][field.name]">
+                      <option ng-repeat="value in field.values" value="{{ value }}"
+                              ng-selected="sc.data[1][field.name] == value">{{ value }}</option>
+                    </select>
+                  </div>
+                  <div class="help-text" ng-if="field.help_text">
+                    {{ field.help_text }}
+                  </div>
+                  <div class="help-text" ng-if="field.help_url">
+                    See <a href="{{ field.help_url }}" target="_blank">Documentation</a> for more information
+                  </div>
+                </td>
+              </tr>
+            </table>
+          </div>
+
+          <div class="add-storage-link" ng-if="canAddStorageConfig()">
+            <a href="javascript:void(0)" ng-click="addStorageConfig()">Add Additional Storage Engine</a>
+          </div>
         </div>
       </div>
     </div>

static/js/core-config-setup.js

@@ -185,12 +185,17 @@ angular.module("core-config-setup", ['angularFileUpload'])
 
         $scope.checkValidateAndSave = function() {
           if ($scope.configform.$valid) {
+            saveStorageConfig();
             $scope.validateAndSave();
             return;
           }
 
-          $element.find("input.ng-invalid:first")[0].scrollIntoView();
-          $element.find("input.ng-invalid:first").focus();
+          var query = $element.find("input.ng-invalid:first");
+
+          if (query && query.length) {
+            query[0].scrollIntoView();
+            query.focus();
+          }
         };
 
         $scope.validateAndSave = function() {
@@ -277,6 +282,99 @@ angular.module("core-config-setup", ['angularFileUpload'])
           }, ApiService.errorDisplay('Could not save configuration. Please report this error.'));
         };
 
+        // Convert storage config to an array
+        var initializeStorageConfig = function($scope) {
+          var config = $scope.config.DISTRIBUTED_STORAGE_CONFIG || {};
+          var defaultLocations = $scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS || [];
+          var preference = $scope.config.DISTRIBUTED_STORAGE_PREFERENCE || [];
+
+          $scope.serverStorageConfig = angular.copy(config);
+          $scope.storageConfig = [];
+
+          Object.keys(config).forEach(function(location) {
+            $scope.storageConfig.push({
+              location: location,
+              defaultLocation: defaultLocations.indexOf(location) >= 0,
+              data: angular.copy(config[location]),
+              error: {},
+            });
+          });
+
+          if (!$scope.storageConfig.length) {
+            $scope.addStorageConfig('default');
+            return;
+          }
+
+          // match DISTRIBUTED_STORAGE_PREFERENCE order first, remaining are
+          // ordered by unicode point value
+          $scope.storageConfig.sort(function(a, b) {
+            var indexA = preference.indexOf(a.location);
+            var indexB = preference.indexOf(b.location);
+
+            if (indexA > -1 && indexB > -1) return indexA < indexB ? -1 : 1;
+            if (indexA > -1) return -1;
+            if (indexB > -1) return 1;
+
+            return a.location < b.location ? -1 : 1;
+          });
+        };
+
+        $scope.allowChangeLocationStorageConfig = function(location) {
+          if (!$scope.serverStorageConfig[location]) { return true };
+
+          // allow user to change location ID if another exists with the same ID
+          return $scope.storageConfig.filter(function(sc) {
+            return sc.location === location;
+          }).length >= 2;
+        };
+
+        $scope.allowRemoveStorageConfig = function(location) {
+          return $scope.storageConfig.length > 1 && $scope.allowChangeLocationStorageConfig(location);
+        };
+
+        $scope.canAddStorageConfig = function() {
+          return $scope.config &&
+            $scope.config.FEATURE_STORAGE_REPLICATION &&
+            $scope.storageConfig &&
+            (!$scope.storageConfig.length || $scope.storageConfig.length < 10);
+        };
+
+        $scope.addStorageConfig = function(location) {
+          var storageType = 'LocalStorage';
+
+          // Use last storage type by default
+          if ($scope.storageConfig.length) {
+            storageType = $scope.storageConfig[$scope.storageConfig.length-1].data[0];
+          }
+
+          $scope.storageConfig.push({
+            location: location || '',
+            defaultLocation: false,
+            data: [storageType, {}],
+            error: {},
+          });
+        };
+
+        $scope.removeStorageConfig = function(sc) {
+          $scope.storageConfig.splice($scope.storageConfig.indexOf(sc), 1);
+        };
+
+        var saveStorageConfig = function() {
+          var config = {};
+          var defaultLocations = [];
+          var preference = [];
+
+          $scope.storageConfig.forEach(function(sc) {
+            config[sc.location] = sc.data;
+            if (sc.defaultLocation) defaultLocations.push(sc.location);
+            preference.push(sc.location);
+          });
+
+          $scope.config.DISTRIBUTED_STORAGE_CONFIG = config;
+          $scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS = defaultLocations;
+          $scope.config.DISTRIBUTED_STORAGE_PREFERENCE = preference;
+        };
+
         var gitlabSelector = function(key) {
           return function(value) {
             if (!value || !$scope.config) { return; }
@@ -378,18 +476,11 @@ angular.module("core-config-setup", ['angularFileUpload'])
         $scope.$watch('mapped.redis.port', redisSetter('port'));
         $scope.$watch('mapped.redis.password', redisSetter('password'));
 
-        // Add a watch to remove any fields not allowed by the current storage configuration.
-        // We have to do this otherwise extra fields (which are not allowed) can end up in the
-        // configuration.
-        $scope.$watch('config.DISTRIBUTED_STORAGE_CONFIG.local[0]', function(value) {
-          // Remove any fields not associated with the current kind.
-          if (!value || !$scope.STORAGE_CONFIG_FIELDS[value]
-              || !$scope.config.DISTRIBUTED_STORAGE_CONFIG
-              || !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local
-              || !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1]) { return; }
-
-          var allowedFields = $scope.STORAGE_CONFIG_FIELDS[value];
-          var configObject = $scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1];
+        // Remove extra extra fields (which are not allowed) from storage config.
+        var updateFields = function(sc) {
+          var type = sc.data[0];
+          var configObject = sc.data[1];
+          var allowedFields = $scope.STORAGE_CONFIG_FIELDS[type];
 
           // Remove any fields not allowed.
           for (var fieldName in configObject) {
@@ -412,8 +503,53 @@ angular.module("core-config-setup", ['angularFileUpload'])
               configObject[allowedFields[i].name] = configObject[allowedFields[i].name] || false;
             }
           }
+        };
+
+        // Validate and update storage config on update.
+        var refreshStorageConfig = function() {
+          if (!$scope.config || !$scope.storageConfig) return;
+
+          var locationCounts = {};
+          var errors = [];
+          var valid = true;
+
+          $scope.storageConfig.forEach(function(sc) {
+            // remove extra fields from storage config
+            updateFields(sc);
+
+            if (!locationCounts[sc.location]) locationCounts[sc.location] = 0;
+            locationCounts[sc.location]++;
+          });
+
+          // validate storage config
+          $scope.storageConfig.forEach(function(sc) {
+            var error = {};
+
+            if ($scope.config.FEATURE_STORAGE_REPLICATION && sc.data[0] === 'LocalStorage') {
+              error.engine = 'Replication to a locally mounted directory is unsupported as it is only accessible on a single machine.';
+              valid = false;
+            }
+
+            if (locationCounts[sc.location] > 1) {
+              error.location = 'Location ID must be unique.';
+              valid = false;
+            }
+
+            errors.push(error);
+          });
+
+          $scope.storageConfigError = errors;
+          $scope.configform.$setValidity('storageConfig', valid);
+        };
+
+        $scope.$watch('config.FEATURE_STORAGE_REPLICATION', function() {
+          refreshStorageConfig();
         });
 
+        $scope.$watch('storageConfig', function() {
+          refreshStorageConfig();
+        }, true);
+
         $scope.$watch('config', function(value) {
           $scope.mapped['$hasChanges'] = true;
         }, true);
@@ -424,6 +560,7 @@ angular.module("core-config-setup", ['angularFileUpload'])
           ApiService.scGetConfig().then(function(resp) {
             $scope.config = resp['config'] || {};
             initializeMappedLogic($scope.config);
+            initializeStorageConfig($scope);
             $scope.mapped['$hasChanges'] = false;
           }, ApiService.errorDisplay('Could not load config'));
         });

storage/distributedstorage.py

@@ -16,6 +16,7 @@ def _location_aware(unbound_func):
     for preferred in self.preferred_locations:
       if preferred in locations:
         storage = self._storages[preferred]
+        break
 
     if not storage:
       storage = self._storages[random.sample(locations, 1)[0]]
@@ -26,10 +27,10 @@ def _location_aware(unbound_func):
 
 
 class DistributedStorage(StoragePaths):
-  def __init__(self, storages, preferred_locations=[], default_locations=[]):
+  def __init__(self, storages, preferred_locations=None, default_locations=None):
     self._storages = dict(storages)
-    self.preferred_locations = list(preferred_locations)
-    self.default_locations = list(default_locations)
+    self.preferred_locations = list(preferred_locations or [])
+    self.default_locations = list(default_locations or [])
 
   @property
   def locations(self):

util/config/configutil.py

@@ -32,15 +32,15 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
 
   # Default storage configuration.
   if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj:
-    config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['local']
+    config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['default']
     config_obj['DISTRIBUTED_STORAGE_CONFIG'] = {
-      'local': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
+      'default': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
     }
 
-    config_obj['USERFILES_LOCATION'] = 'local'
+    config_obj['USERFILES_LOCATION'] = 'default'
     config_obj['USERFILES_PATH'] = 'userfiles/'
 
-    config_obj['LOG_ARCHIVE_LOCATION'] = 'local'
+    config_obj['LOG_ARCHIVE_LOCATION'] = 'default'
 
   if not 'SERVER_HOSTNAME' in config_obj:
     config_obj['SERVER_HOSTNAME'] = hostname

util/config/database.py

@@ -0,0 +1,9 @@
+from data import model
+
+
+def sync_database_with_config(config):
+  """ This ensures all implicitly required reference table entries exist in the database. """
+
+  location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
+  if location_names:
+    model.image.ensure_image_locations(*location_names)

util/config/validator.py

@@ -30,12 +30,18 @@ JWT_FILENAMES = ['jwt-authn.cert']
 
 CONFIG_FILENAMES = SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES
 
-def get_storage_provider(config):
-  parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}])
+def get_storage_providers(config):
+  storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
+
+  drivers = {}
+
   try:
-    return get_storage_driver(parameters)
+    for name, parameters in storage_config.items():
+      drivers[name] = (parameters[0], get_storage_driver(parameters))
   except TypeError:
-    raise Exception('Missing required storage configuration parameter(s)')
+    raise Exception('Missing required storage configuration parameter(s): %s' % name)
+
+  return drivers
 
 def validate_service_for_config(service, config, password=None):
   """ Attempts to validate the configuration for the given service. """
@@ -80,20 +86,29 @@ def _validate_redis(config, _):
 
 def _validate_registry_storage(config, _):
   """ Validates registry storage. """
-  driver = get_storage_provider(config)
+  replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
 
-  # Run custom validation on the driver.
-  driver.validate(app.config['HTTPCLIENT'])
+  providers = get_storage_providers(config).items()
 
-  # Put and remove a temporary file to make sure the normal storage paths work.
-  driver.put_content('_verify', 'testing 123')
-  driver.remove('_verify')
+  if not providers:
+    raise Exception('Storage configuration required')
 
-  # Run setup on the driver if the read/write succeeded.
-  try:
-    driver.setup()
-  except Exception as ex:
-    raise Exception('Could not prepare storage: %s' % str(ex))
+  for name, (storage_type, driver) in providers:
+    try:
+      if replication_enabled and storage_type == 'LocalStorage':
+        raise Exception('Locally mounted directory not supported with storage replication')
+
+      # Run custom validation on the driver.
+      driver.validate(app.config['HTTPCLIENT'])
+
+      # Put and remove a temporary file to make sure the normal storage paths work.
+      driver.put_content('_verify', 'testing 123')
+      driver.remove('_verify')
+
+      # Run setup on the driver if the read/write succeeded.
+      driver.setup()
+    except Exception as ex:
+      raise Exception('Invalid storage configuration: %s: %s' % (name, str(ex)))
 
 
 def _validate_mailing(config, _):