vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

superuser: add storage replication config

Browse source
This commit is contained in:
Silas Sewell 2015-10-26 16:06:05 -07:00
parent c33ae0e896
commit 5000b1621c
15 changed files with 357 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -43,7 +43,7 @@ ADD conf/init/doupdatelimits.sh /etc/my_init.d/
ADD conf/init/copy_syslog_config.sh /etc/my_init.d/
ADD conf/init/runmigration.sh /etc/my_init.d/
ADD conf/init/syslog-ng.conf /etc/syslog-ng/
ADD conf/init/zz_release.sh /etc/my_init.d/
ADD conf/init/zz_boot.sh /etc/my_init.d/
ADD conf/init/service/ /etc/service/

20
boot.py Normal file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env python
import release
from app import app
from data.model.release import set_region_release
from util.config.database import sync_database_with_config
def main():
if app.config.get('SETUP_COMPLETE', False):
sync_database_with_config(app.config)
# Record deploy
if release.REGION and release.GIT_HEAD:
set_region_release(release.SERVICE, release.REGION, release.GIT_HEAD)
if __name__ == '__main__':
main()

3
conf/init/zz_boot.sh Executable file
View file

@ -0,0 +1,3 @@
#!/bin/bash
/venv/bin/python /boot.py

8
conf/init/zz_release.sh
View file

@ -1,8 +0,0 @@
#!/bin/bash
set -e
source venv/bin/activate
export PYTHONPATH=.
python /release.py

16
data/model/image.py
View file

@ -356,3 +356,19 @@ def get_image(repo, dockerfile_id):
return Image.get(Image.docker_image_id == dockerfile_id, Image.repository == repo)
except Image.DoesNotExist:
return None
def ensure_image_locations(*names):
with db_transaction():
locations = ImageStorageLocation.select().where(ImageStorageLocation.name << names)
insert_names = list(names)
for location in locations:
insert_names.remove(location.name)
if not insert_names:
return
data = [{'name': name} for name in insert_names]
ImageStorageLocation.insert_many(data).execute()

4
endpoints/api/suconfig.py
View file

@ -16,6 +16,7 @@ from auth.permissions import SuperUserPermission
from auth.auth_context import get_authenticated_user
from data.database import User
from util.config.configutil import add_enterprise_config_defaults
from util.config.database import sync_database_with_config
from util.config.validator import validate_service_for_config, CONFIG_FILENAMES
from data.runmigration import run_alembic_migration
from data.users import get_federated_service_name
@ -216,6 +217,9 @@ class SuperUserConfig(ApiResource):
current_user = get_authenticated_user()
model.user.confirm_attached_federated_login(current_user, service_name)
# Ensure database is up-to-date with config
sync_database_with_config(config_object)
return {
'exists': True,
'config': config_object

12
release.py
View file

@ -12,15 +12,3 @@ REGION = os.environ.get('QUAY_REGION')
if os.path.isfile(_GIT_HEAD_PATH):
with open(_GIT_HEAD_PATH) as f:
GIT_HEAD = f.read().strip()
def main():
from app import app
from data.model.release import set_region_release
if REGION and GIT_HEAD:
set_region_release(SERVICE, REGION, GIT_HEAD)
if __name__ == '__main__':
main()

26
static/css/pages/superuser.css
View file

@ -23,3 +23,29 @@
.super-user .user-row.disabled .avatar {
-webkit-filter: grayscale(100%);
}
.super-user td .co-alert {
margin: 16px 0 0 0;
}
.super-user .add-storage-link {
margin-top: 5px;
}
.super-user .storage-config {
border-bottom: 1px solid #eee;
padding: 0 0 10px 0;
margin: 10px 0 0 0;
}
.super-user .last {
border-bottom: none;
}
.super-user .feature-storage-replication {
margin: 15px 0 10px 0;
}
.super-user .input-util {
margin-top: 10px;
}

2
static/css/quay.css
View file

@ -1363,7 +1363,7 @@ i.toggle-icon:hover {
form input.ng-invalid.ng-dirty,
*[ng-form] input.ng-invalid.ng-dirty {
background-color: #FDD7D9;
background-color: #FDD7D9 !important;
}
form input.ng-valid.ng-dirty,

54
static/directives/config/config-setup-tool.html
View file

@ -194,42 +194,78 @@
<strong>A remote storage system is required for high-avaliability systems.</strong>
</p>
<div class="co-checkbox feature-storage-replication">
<input id="ftsr" type="checkbox" ng-model="config.FEATURE_STORAGE_REPLICATION">
<label for="ftsr">Enable Storage Replication</label>
<!-- TODO(josephschorr): add link to documentation -->
<div class="help-text">If enabled, replicates storage to other regions.</div>
</div>
<div class="storage-config" ng-class="$last ? 'last' : ''" ng-repeat="sc in storageConfig">
<table class="config-table">
<tr>
<td class="non-input">Location ID:</td>
<td>
<input class="form-control" ng-if="allowChangeLocationStorageConfig(sc.location)" ng-class="storageConfigError[$index].location ? 'ng-invalid' : ''" ng-model="sc.location" ng-pattern="/^[a-zA-Z0-9_-]+$/" required>
<div ng-if="!allowChangeLocationStorageConfig(sc.location)">
{{ sc.location }}
</div>
<div class="co-alert co-alert-danger" ng-show="storageConfigError[$index].location">
{{ storageConfigError[$index].location }}
</div>
<div class="input-util" ng-if="allowRemoveStorageConfig(sc.location)"><a href="javascript:void(0)" class="remove-link" ng-click="removeStorageConfig(sc)">Remove</a></div>
</td>
</tr>
<tr ng-if="config.FEATURE_STORAGE_REPLICATION">
<td class="non-input">Set Default:</td>
<td>
<div class="co-checkbox">
<input id="default-location-{{ $index }}" ng-model="sc.defaultLocation" type="checkbox">
<label for="default-location-{{ $index }}">Replicate to storage engine by default</label>
</div>
</td>
</tr>
<tr>
<td class="non-input">Storage Engine:</td>
<td>
<select ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[0]">
<select class="form-control" ng-class="storageConfigError[$index].engine ? 'ng-invalid' : ''" ng-model="sc.data[0]">
<option value="LocalStorage">Locally mounted directory</option>
<option value="S3Storage">Amazon S3</option>
<option value="GoogleCloudStorage">Google Cloud Storage</option>
<option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
<option value="SwiftStorage">OpenStack Storage (Swift)</option>
</select>
<div class="co-alert co-alert-danger" ng-if="storageConfigError[$index].engine">
{{ storageConfigError[$index].engine }}
</div>
</td>
</tr>
<!-- Fields -->
<tr ng-repeat="field in STORAGE_CONFIG_FIELDS[config.DISTRIBUTED_STORAGE_CONFIG.local[0]]">
<tr ng-repeat="field in STORAGE_CONFIG_FIELDS[sc.data[0]]">
<td>{{ field.title }}:</td>
<td>
<span class="config-map-field"
binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
binding="sc.data[1][field.name]"
ng-if="field.kind == 'map'"
keys="field.keys"></span>
<span class="config-string-field"
binding="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]"
binding="sc.data[1][field.name]"
placeholder="{{ field.placeholder }}"
ng-if="field.kind == 'text'"
is-optional="field.optional"></span>
<div class="co-checkbox" ng-if="field.kind == 'bool'">
<input id="dsc-{{ field.name }}" type="checkbox"
ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
ng-model="sc.data[1][field.name]">
<label for="dsc-{{ field.name }}">{{ field.placeholder }}</label>
</div>
<div ng-if="field.kind == 'option'">
<select ng-model="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name]">
<select ng-model="sc.data[1][field.name]">
<option ng-repeat="value in field.values" value="{{ value }}"
ng-selected="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name] == value">{{ value }}</option>
ng-selected="sc.data[1][field.name] == value">{{ value }}</option>
</select>
</div>
<div class="help-text" ng-if="field.help_text">
@ -241,7 +277,11 @@
</td>
</tr>
</table>
</div>
<div class="add-storage-link" ng-if="canAddStorageConfig()">
<a href="javascript:void(0)" ng-click="addStorageConfig()">Add Additional Storage Engine</a>
</div>
</div>
</div>
</div>

165
static/js/core-config-setup.js
View file

@ -185,12 +185,17 @@ angular.module("core-config-setup", ['angularFileUpload'])
$scope.checkValidateAndSave = function() {
if ($scope.configform.$valid) {
saveStorageConfig();
$scope.validateAndSave();
return;
}
$element.find("input.ng-invalid:first")[0].scrollIntoView();
$element.find("input.ng-invalid:first").focus();
var query = $element.find("input.ng-invalid:first");
if (query && query.length) {
query[0].scrollIntoView();
query.focus();
}
};
$scope.validateAndSave = function() {
@ -277,6 +282,99 @@ angular.module("core-config-setup", ['angularFileUpload'])
}, ApiService.errorDisplay('Could not save configuration. Please report this error.'));
};
// Convert storage config to an array
var initializeStorageConfig = function($scope) {
var config = $scope.config.DISTRIBUTED_STORAGE_CONFIG || {};
var defaultLocations = $scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS || [];
var preference = $scope.config.DISTRIBUTED_STORAGE_PREFERENCE || [];
$scope.serverStorageConfig = angular.copy(config);
$scope.storageConfig = [];
Object.keys(config).forEach(function(location) {
$scope.storageConfig.push({
location: location,
defaultLocation: defaultLocations.indexOf(location) >= 0,
data: angular.copy(config[location]),
error: {},
});
});
if (!$scope.storageConfig.length) {
$scope.addStorageConfig('default');
return;
}
// match DISTRIBUTED_STORAGE_PREFERENCE order first, remaining are
// ordered by unicode point value
$scope.storageConfig.sort(function(a, b) {
var indexA = preference.indexOf(a.location);
var indexB = preference.indexOf(b.location);
if (indexA > -1 && indexB > -1) return indexA < indexB ? -1 : 1;
if (indexA > -1) return -1;
if (indexB > -1) return 1;
return a.location < b.location ? -1 : 1;
});
};
$scope.allowChangeLocationStorageConfig = function(location) {
if (!$scope.serverStorageConfig[location]) { return true };
// allow user to change location ID if another exists with the same ID
return $scope.storageConfig.filter(function(sc) {
return sc.location === location;
}).length >= 2;
};
$scope.allowRemoveStorageConfig = function(location) {
return $scope.storageConfig.length > 1 && $scope.allowChangeLocationStorageConfig(location);
};
$scope.canAddStorageConfig = function() {
return $scope.config &&
$scope.config.FEATURE_STORAGE_REPLICATION &&
$scope.storageConfig &&
(!$scope.storageConfig.length || $scope.storageConfig.length < 10);
};
$scope.addStorageConfig = function(location) {
var storageType = 'LocalStorage';
// Use last storage type by default
if ($scope.storageConfig.length) {
storageType = $scope.storageConfig[$scope.storageConfig.length-1].data[0];
}
$scope.storageConfig.push({
location: location || '',
defaultLocation: false,
data: [storageType, {}],
error: {},
});
};
$scope.removeStorageConfig = function(sc) {
$scope.storageConfig.splice($scope.storageConfig.indexOf(sc), 1);
};
var saveStorageConfig = function() {
var config = {};
var defaultLocations = [];
var preference = [];
$scope.storageConfig.forEach(function(sc) {
config[sc.location] = sc.data;
if (sc.defaultLocation) defaultLocations.push(sc.location);
preference.push(sc.location);
});
$scope.config.DISTRIBUTED_STORAGE_CONFIG = config;
$scope.config.DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS = defaultLocations;
$scope.config.DISTRIBUTED_STORAGE_PREFERENCE = preference;
};
var gitlabSelector = function(key) {
return function(value) {
if (!value || !$scope.config) { return; }
@ -378,18 +476,11 @@ angular.module("core-config-setup", ['angularFileUpload'])
$scope.$watch('mapped.redis.port', redisSetter('port'));
$scope.$watch('mapped.redis.password', redisSetter('password'));
// Add a watch to remove any fields not allowed by the current storage configuration.
// We have to do this otherwise extra fields (which are not allowed) can end up in the
// configuration.
$scope.$watch('config.DISTRIBUTED_STORAGE_CONFIG.local[0]', function(value) {
// Remove any fields not associated with the current kind.
if (!value || !$scope.STORAGE_CONFIG_FIELDS[value]
|| !$scope.config.DISTRIBUTED_STORAGE_CONFIG
|| !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local
|| !$scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1]) { return; }
var allowedFields = $scope.STORAGE_CONFIG_FIELDS[value];
var configObject = $scope.config.DISTRIBUTED_STORAGE_CONFIG.local[1];
// Remove extra extra fields (which are not allowed) from storage config.
var updateFields = function(sc) {
var type = sc.data[0];
var configObject = sc.data[1];
var allowedFields = $scope.STORAGE_CONFIG_FIELDS[type];
// Remove any fields not allowed.
for (var fieldName in configObject) {
@ -412,8 +503,53 @@ angular.module("core-config-setup", ['angularFileUpload'])
configObject[allowedFields[i].name] = configObject[allowedFields[i].name] || false;
}
}
};
// Validate and update storage config on update.
var refreshStorageConfig = function() {
if (!$scope.config || !$scope.storageConfig) return;
var locationCounts = {};
var errors = [];
var valid = true;
$scope.storageConfig.forEach(function(sc) {
// remove extra fields from storage config
updateFields(sc);
if (!locationCounts[sc.location]) locationCounts[sc.location] = 0;
locationCounts[sc.location]++;
});
// validate storage config
$scope.storageConfig.forEach(function(sc) {
var error = {};
if ($scope.config.FEATURE_STORAGE_REPLICATION && sc.data[0] === 'LocalStorage') {
error.engine = 'Replication to a locally mounted directory is unsupported as it is only accessible on a single machine.';
valid = false;
}
if (locationCounts[sc.location] > 1) {
error.location = 'Location ID must be unique.';
valid = false;
}
errors.push(error);
});
$scope.storageConfigError = errors;
$scope.configform.$setValidity('storageConfig', valid);
};
$scope.$watch('config.FEATURE_STORAGE_REPLICATION', function() {
refreshStorageConfig();
});
$scope.$watch('storageConfig', function() {
refreshStorageConfig();
}, true);
$scope.$watch('config', function(value) {
$scope.mapped['$hasChanges'] = true;
}, true);
@ -424,6 +560,7 @@ angular.module("core-config-setup", ['angularFileUpload'])
ApiService.scGetConfig().then(function(resp) {
$scope.config = resp['config'] || {};
initializeMappedLogic($scope.config);
initializeStorageConfig($scope);
$scope.mapped['$hasChanges'] = false;
}, ApiService.errorDisplay('Could not load config'));
});

7
storage/distributedstorage.py
View file

@ -16,6 +16,7 @@ def _location_aware(unbound_func):
for preferred in self.preferred_locations:
if preferred in locations:
storage = self._storages[preferred]
break
if not storage:
storage = self._storages[random.sample(locations, 1)[0]]
@ -26,10 +27,10 @@ def _location_aware(unbound_func):
class DistributedStorage(StoragePaths):
def __init__(self, storages, preferred_locations=[], default_locations=[]):
def __init__(self, storages, preferred_locations=None, default_locations=None):
self._storages = dict(storages)
self.preferred_locations = list(preferred_locations)
self.default_locations = list(default_locations)
self.preferred_locations = list(preferred_locations or [])
self.default_locations = list(default_locations or [])
@property
def locations(self):

8
util/config/configutil.py
View file

@ -32,15 +32,15 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
# Default storage configuration.
if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj:
config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['local']
config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['default']
config_obj['DISTRIBUTED_STORAGE_CONFIG'] = {
'local': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
'default': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
}
config_obj['USERFILES_LOCATION'] = 'local'
config_obj['USERFILES_LOCATION'] = 'default'
config_obj['USERFILES_PATH'] = 'userfiles/'
config_obj['LOG_ARCHIVE_LOCATION'] = 'local'
config_obj['LOG_ARCHIVE_LOCATION'] = 'default'
if not 'SERVER_HOSTNAME' in config_obj:
config_obj['SERVER_HOSTNAME'] = hostname

9
util/config/database.py Normal file
View file

@ -0,0 +1,9 @@
from data import model
def sync_database_with_config(config):
""" This ensures all implicitly required reference table entries exist in the database. """
location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
if location_names:
model.image.ensure_image_locations(*location_names)

29
util/config/validator.py
View file

@ -30,12 +30,18 @@ JWT_FILENAMES = ['jwt-authn.cert']
CONFIG_FILENAMES = SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES
def get_storage_provider(config):
parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}])
def get_storage_providers(config):
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
drivers = {}
try:
return get_storage_driver(parameters)
for name, parameters in storage_config.items():
drivers[name] = (parameters[0], get_storage_driver(parameters))
except TypeError:
raise Exception('Missing required storage configuration parameter(s)')
raise Exception('Missing required storage configuration parameter(s): %s' % name)
return drivers
def validate_service_for_config(service, config, password=None):
""" Attempts to validate the configuration for the given service. """
@ -80,7 +86,17 @@ def _validate_redis(config, _):
def _validate_registry_storage(config, _):
""" Validates registry storage. """
driver = get_storage_provider(config)
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
providers = get_storage_providers(config).items()
if not providers:
raise Exception('Storage configuration required')
for name, (storage_type, driver) in providers:
try:
if replication_enabled and storage_type == 'LocalStorage':
raise Exception('Locally mounted directory not supported with storage replication')
# Run custom validation on the driver.
driver.validate(app.config['HTTPCLIENT'])
@ -90,10 +106,9 @@ def _validate_registry_storage(config, _):
driver.remove('_verify')
# Run setup on the driver if the read/write succeeded.
try:
driver.setup()
except Exception as ex:
raise Exception('Could not prepare storage: %s' % str(ex))
raise Exception('Invalid storage configuration: %s: %s' % (name, str(ex)))
def _validate_mailing(config, _):