From 5000b1621cee4052c0bad68f981913c8cbb975e7 Mon Sep 17 00:00:00 2001 From: Silas Sewell Date: Mon, 26 Oct 2015 16:06:05 -0700 Subject: [PATCH] superuser: add storage replication config --- Dockerfile | 2 +- boot.py | 20 +++ conf/init/zz_boot.sh | 3 + conf/init/zz_release.sh | 8 - data/model/image.py | 16 ++ endpoints/api/suconfig.py | 6 +- release.py | 12 -- static/css/pages/superuser.css | 28 ++- static/css/quay.css | 2 +- .../directives/config/config-setup-tool.html | 130 +++++++++----- static/js/core-config-setup.js | 167 ++++++++++++++++-- storage/distributedstorage.py | 7 +- util/config/configutil.py | 8 +- util/config/database.py | 9 + util/config/validator.py | 45 +++-- 15 files changed, 357 insertions(+), 106 deletions(-) create mode 100644 boot.py create mode 100755 conf/init/zz_boot.sh delete mode 100755 conf/init/zz_release.sh create mode 100644 util/config/database.py diff --git a/Dockerfile b/Dockerfile index 5a757aa21..737ae4934 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ ADD conf/init/doupdatelimits.sh /etc/my_init.d/ ADD conf/init/copy_syslog_config.sh /etc/my_init.d/ ADD conf/init/runmigration.sh /etc/my_init.d/ ADD conf/init/syslog-ng.conf /etc/syslog-ng/ -ADD conf/init/zz_release.sh /etc/my_init.d/ +ADD conf/init/zz_boot.sh /etc/my_init.d/ ADD conf/init/service/ /etc/service/ diff --git a/boot.py b/boot.py new file mode 100644 index 000000000..f78ad6ab8 --- /dev/null +++ b/boot.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +import release + +from app import app +from data.model.release import set_region_release +from util.config.database import sync_database_with_config + + +def main(): + if app.config.get('SETUP_COMPLETE', False): + sync_database_with_config(app.config) + + # Record deploy + if release.REGION and release.GIT_HEAD: + set_region_release(release.SERVICE, release.REGION, release.GIT_HEAD) + + +if __name__ == '__main__': + main() diff --git a/conf/init/zz_boot.sh b/conf/init/zz_boot.sh new file mode 100755 index 000000000..ab760266b --- /dev/null +++ b/conf/init/zz_boot.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +/venv/bin/python /boot.py diff --git a/conf/init/zz_release.sh b/conf/init/zz_release.sh deleted file mode 100755 index 152494cff..000000000 --- a/conf/init/zz_release.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -source venv/bin/activate - -export PYTHONPATH=. - -python /release.py diff --git a/data/model/image.py b/data/model/image.py index 078875417..f084f2662 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -356,3 +356,19 @@ def get_image(repo, dockerfile_id): return Image.get(Image.docker_image_id == dockerfile_id, Image.repository == repo) except Image.DoesNotExist: return None + + +def ensure_image_locations(*names): + with db_transaction(): + locations = ImageStorageLocation.select().where(ImageStorageLocation.name << names) + + insert_names = list(names) + + for location in locations: + insert_names.remove(location.name) + + if not insert_names: + return + + data = [{'name': name} for name in insert_names] + ImageStorageLocation.insert_many(data).execute() diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index aaea5a309..5a5cd9346 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -16,6 +16,7 @@ from auth.permissions import SuperUserPermission from auth.auth_context import get_authenticated_user from data.database import User from util.config.configutil import add_enterprise_config_defaults +from util.config.database import sync_database_with_config from util.config.validator import validate_service_for_config, CONFIG_FILENAMES from data.runmigration import run_alembic_migration from data.users import get_federated_service_name @@ -216,6 +217,9 @@ class SuperUserConfig(ApiResource): current_user = get_authenticated_user() model.user.confirm_attached_federated_login(current_user, service_name) + # Ensure database is up-to-date with config + sync_database_with_config(config_object) + return { 'exists': True, 'config': config_object @@ -373,4 +377,4 @@ class SuperUserConfigValidate(ApiResource): config = request.get_json()['config'] return validate_service_for_config(service, config, request.get_json().get('password', '')) - abort(403) \ No newline at end of file + abort(403) diff --git a/release.py b/release.py index 91a46f796..a0439d9a8 100644 --- a/release.py +++ b/release.py @@ -12,15 +12,3 @@ REGION = os.environ.get('QUAY_REGION') if os.path.isfile(_GIT_HEAD_PATH): with open(_GIT_HEAD_PATH) as f: GIT_HEAD = f.read().strip() - - -def main(): - from app import app - from data.model.release import set_region_release - - if REGION and GIT_HEAD: - set_region_release(SERVICE, REGION, GIT_HEAD) - - -if __name__ == '__main__': - main() diff --git a/static/css/pages/superuser.css b/static/css/pages/superuser.css index 380e31879..d1eae17bd 100644 --- a/static/css/pages/superuser.css +++ b/static/css/pages/superuser.css @@ -22,4 +22,30 @@ .super-user .user-row.disabled .avatar { -webkit-filter: grayscale(100%); -} \ No newline at end of file +} + +.super-user td .co-alert { + margin: 16px 0 0 0; +} + +.super-user .add-storage-link { + margin-top: 5px; +} + +.super-user .storage-config { + border-bottom: 1px solid #eee; + padding: 0 0 10px 0; + margin: 10px 0 0 0; +} + +.super-user .last { + border-bottom: none; +} + +.super-user .feature-storage-replication { + margin: 15px 0 10px 0; +} + +.super-user .input-util { + margin-top: 10px; +} diff --git a/static/css/quay.css b/static/css/quay.css index bd8e3873a..143496034 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -1363,7 +1363,7 @@ i.toggle-icon:hover { form input.ng-invalid.ng-dirty, *[ng-form] input.ng-invalid.ng-dirty { - background-color: #FDD7D9; + background-color: #FDD7D9 !important; } form input.ng-valid.ng-dirty, diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index 9edbffd75..a36d08416 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -194,54 +194,94 @@ A remote storage system is required for high-avaliability systems.

- - - - - +
+ + + +
If enabled, replicates storage to other regions.
+
- - - - + + + + + + + +
Storage Engine: - -
{{ field.title }}: - - -
- - -
-
- + + + + + + + + + + + + + - -
Location ID: + +
+ {{ sc.location }} +
+
+ {{ storageConfigError[$index].location }} +
+
Remove
+
Set Default: +
+ + +
+
Storage Engine: + - -
- {{ field.help_text }} -
-
- See Documentation for more information -
-
+
+ {{ storageConfigError[$index].engine }} +
+
{{ field.title }}: + + +
+ + +
+
+ +
+
+ {{ field.help_text }} +
+
+ See Documentation for more information +
+
+ + +
+ Add Additional Storage Engine +
diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index e3d5c3483..13b49b865 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -185,12 +185,17 @@ angular.module("core-config-setup", ['angularFileUpload']) $scope.checkValidateAndSave = function() { if ($scope.configform.$valid) { + saveStorageConfig(); $scope.validateAndSave(); return; } - $element.find("input.ng-invalid:first")[0].scrollIntoView(); - $element.find("input.ng-invalid:first").focus(); + var query = $element.find("input.ng-invalid:first"); + + if (query && query.length) { + query[0].scrollIntoView(); + query.focus(); + } }; $scope.validateAndSave = function() { @@ -277,6 +282,99 @@ angular.module("core-config-setup", ['angularFileUpload']) }, ApiService.errorDisplay('Could not save configuration. Please report this error.')); }; + // Convert storage config to an array + var initializeStorageConfig = function($scope) { + var config = $scope.config.DISTRIBUTED_STORAGE_CONFIG || {}; + var defaultLocations = $scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS || []; + var preference = $scope.config.DISTRIBUTED_STORAGE_PREFERENCE || []; + + $scope.serverStorageConfig = angular.copy(config); + $scope.storageConfig = []; + + Object.keys(config).forEach(function(location) { + $scope.storageConfig.push({ + location: location, + defaultLocation: defaultLocations.indexOf(location) >= 0, + data: angular.copy(config[location]), + error: {}, + }); + }); + + if (!$scope.storageConfig.length) { + $scope.addStorageConfig('default'); + return; + } + + // match DISTRIBUTED_STORAGE_PREFERENCE order first, remaining are + // ordered by unicode point value + $scope.storageConfig.sort(function(a, b) { + var indexA = preference.indexOf(a.location); + var indexB = preference.indexOf(b.location); + + if (indexA > -1 && indexB > -1) return indexA < indexB ? -1 : 1; + if (indexA > -1) return -1; + if (indexB > -1) return 1; + + return a.location < b.location ? -1 : 1; + }); + }; + + $scope.allowChangeLocationStorageConfig = function(location) { + if (!$scope.serverStorageConfig[location]) { return true }; + + // allow user to change location ID if another exists with the same ID + return $scope.storageConfig.filter(function(sc) { + return sc.location === location; + }).length >= 2; + }; + + $scope.allowRemoveStorageConfig = function(location) { + return $scope.storageConfig.length > 1 && $scope.allowChangeLocationStorageConfig(location); + }; + + $scope.canAddStorageConfig = function() { + return $scope.config && + $scope.config.FEATURE_STORAGE_REPLICATION && + $scope.storageConfig && + (!$scope.storageConfig.length || $scope.storageConfig.length < 10); + }; + + $scope.addStorageConfig = function(location) { + var storageType = 'LocalStorage'; + + // Use last storage type by default + if ($scope.storageConfig.length) { + storageType = $scope.storageConfig[$scope.storageConfig.length-1].data[0]; + } + + $scope.storageConfig.push({ + location: location || '', + defaultLocation: false, + data: [storageType, {}], + error: {}, + }); + }; + + $scope.removeStorageConfig = function(sc) { + $scope.storageConfig.splice($scope.storageConfig.indexOf(sc), 1); + }; + + var saveStorageConfig = function() { + var config = {}; + var defaultLocations = []; + var preference = []; + + $scope.storageConfig.forEach(function(sc) { + config[sc.location] = sc.data; + if (sc.defaultLocation) defaultLocations.push(sc.location); + preference.push(sc.location); + }); + + $scope.config.DISTRIBUTED_STORAGE_CONFIG = config; + $scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS = defaultLocations; + $scope.config.DISTRIBUTED_STORAGE_PREFERENCE = preference; + }; + var gitlabSelector = function(key) { return function(value) { if (!value || !$scope.config) { return; } @@ -378,18 +476,11 @@ angular.module("core-config-setup", ['angularFileUpload']) $scope.$watch('mapped.redis.port', redisSetter('port')); $scope.$watch('mapped.redis.password', redisSetter('password')); - // Add a watch to remove any fields not allowed by the current storage configuration. - // We have to do this otherwise extra fields (which are not allowed) can end up in the - // configuration. - $scope.$watch('config.DISTRIBUTED_STORAGE_CONFIG.local[0]', function(value) { - // Remove any fields not associated with the current kind. - if (!value || !$scope.STORAGE_CONFIG_FIELDS[value] - || !$scope.config.DISTRIBUTED_STORAGE_CONFIG - || !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local - || !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1]) { return; } - - var allowedFields = $scope.STORAGE_CONFIG_FIELDS[value]; - var configObject = $scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1]; + // Remove extra extra fields (which are not allowed) from storage config. + var updateFields = function(sc) { + var type = sc.data[0]; + var configObject = sc.data[1]; + var allowedFields = $scope.STORAGE_CONFIG_FIELDS[type]; // Remove any fields not allowed. for (var fieldName in configObject) { @@ -412,8 +503,53 @@ angular.module("core-config-setup", ['angularFileUpload']) configObject[allowedFields[i].name] = configObject[allowedFields[i].name] || false; } } + }; + + // Validate and update storage config on update. + var refreshStorageConfig = function() { + if (!$scope.config || !$scope.storageConfig) return; + + var locationCounts = {}; + var errors = []; + var valid = true; + + $scope.storageConfig.forEach(function(sc) { + // remove extra fields from storage config + updateFields(sc); + + if (!locationCounts[sc.location]) locationCounts[sc.location] = 0; + locationCounts[sc.location]++; + }); + + // validate storage config + $scope.storageConfig.forEach(function(sc) { + var error = {}; + + if ($scope.config.FEATURE_STORAGE_REPLICATION && sc.data[0] === 'LocalStorage') { + error.engine = 'Replication to a locally mounted directory is unsupported as it is only accessible on a single machine.'; + valid = false; + } + + if (locationCounts[sc.location] > 1) { + error.location = 'Location ID must be unique.'; + valid = false; + } + + errors.push(error); + }); + + $scope.storageConfigError = errors; + $scope.configform.$setValidity('storageConfig', valid); + }; + + $scope.$watch('config.FEATURE_STORAGE_REPLICATION', function() { + refreshStorageConfig(); }); + $scope.$watch('storageConfig', function() { + refreshStorageConfig(); + }, true); + $scope.$watch('config', function(value) { $scope.mapped['$hasChanges'] = true; }, true); @@ -424,6 +560,7 @@ angular.module("core-config-setup", ['angularFileUpload']) ApiService.scGetConfig().then(function(resp) { $scope.config = resp['config'] || {}; initializeMappedLogic($scope.config); + initializeStorageConfig($scope); $scope.mapped['$hasChanges'] = false; }, ApiService.errorDisplay('Could not load config')); }); @@ -919,4 +1056,4 @@ angular.module("core-config-setup", ['angularFileUpload']) } }; return directiveDefinitionObject; - }); \ No newline at end of file + }); diff --git a/storage/distributedstorage.py b/storage/distributedstorage.py index 49f32c559..55afec293 100644 --- a/storage/distributedstorage.py +++ b/storage/distributedstorage.py @@ -16,6 +16,7 @@ def _location_aware(unbound_func): for preferred in self.preferred_locations: if preferred in locations: storage = self._storages[preferred] + break if not storage: storage = self._storages[random.sample(locations, 1)[0]] @@ -26,10 +27,10 @@ def _location_aware(unbound_func): class DistributedStorage(StoragePaths): - def __init__(self, storages, preferred_locations=[], default_locations=[]): + def __init__(self, storages, preferred_locations=None, default_locations=None): self._storages = dict(storages) - self.preferred_locations = list(preferred_locations) - self.default_locations = list(default_locations) + self.preferred_locations = list(preferred_locations or []) + self.default_locations = list(default_locations or []) @property def locations(self): diff --git a/util/config/configutil.py b/util/config/configutil.py index 2a007a794..5b9bc1dcb 100644 --- a/util/config/configutil.py +++ b/util/config/configutil.py @@ -32,15 +32,15 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname): # Default storage configuration. if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj: - config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['local'] + config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['default'] config_obj['DISTRIBUTED_STORAGE_CONFIG'] = { - 'local': ['LocalStorage', {'storage_path': '/datastorage/registry'}] + 'default': ['LocalStorage', {'storage_path': '/datastorage/registry'}] } - config_obj['USERFILES_LOCATION'] = 'local' + config_obj['USERFILES_LOCATION'] = 'default' config_obj['USERFILES_PATH'] = 'userfiles/' - config_obj['LOG_ARCHIVE_LOCATION'] = 'local' + config_obj['LOG_ARCHIVE_LOCATION'] = 'default' if not 'SERVER_HOSTNAME' in config_obj: config_obj['SERVER_HOSTNAME'] = hostname diff --git a/util/config/database.py b/util/config/database.py new file mode 100644 index 000000000..ea04dc5dc --- /dev/null +++ b/util/config/database.py @@ -0,0 +1,9 @@ +from data import model + + +def sync_database_with_config(config): + """ This ensures all implicitly required reference table entries exist in the database. """ + + location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys() + if location_names: + model.image.ensure_image_locations(*location_names) diff --git a/util/config/validator.py b/util/config/validator.py index d4dc1cd39..1e8846646 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -30,12 +30,18 @@ JWT_FILENAMES = ['jwt-authn.cert'] CONFIG_FILENAMES = SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES -def get_storage_provider(config): - parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}]) +def get_storage_providers(config): + storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {}) + + drivers = {} + try: - return get_storage_driver(parameters) + for name, parameters in storage_config.items(): + drivers[name] = (parameters[0], get_storage_driver(parameters)) except TypeError: - raise Exception('Missing required storage configuration parameter(s)') + raise Exception('Missing required storage configuration parameter(s): %s' % name) + + return drivers def validate_service_for_config(service, config, password=None): """ Attempts to validate the configuration for the given service. """ @@ -80,20 +86,29 @@ def _validate_redis(config, _): def _validate_registry_storage(config, _): """ Validates registry storage. """ - driver = get_storage_provider(config) + replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False) - # Run custom validation on the driver. - driver.validate(app.config['HTTPCLIENT']) + providers = get_storage_providers(config).items() - # Put and remove a temporary file to make sure the normal storage paths work. - driver.put_content('_verify', 'testing 123') - driver.remove('_verify') + if not providers: + raise Exception('Storage configuration required') - # Run setup on the driver if the read/write succeeded. - try: - driver.setup() - except Exception as ex: - raise Exception('Could not prepare storage: %s' % str(ex)) + for name, (storage_type, driver) in providers: + try: + if replication_enabled and storage_type == 'LocalStorage': + raise Exception('Locally mounted directory not supported with storage replication') + + # Run custom validation on the driver. + driver.validate(app.config['HTTPCLIENT']) + + # Put and remove a temporary file to make sure the normal storage paths work. + driver.put_content('_verify', 'testing 123') + driver.remove('_verify') + + # Run setup on the driver if the read/write succeeded. + driver.setup() + except Exception as ex: + raise Exception('Invalid storage configuration: %s: %s' % (name, str(ex))) def _validate_mailing(config, _):