diff --git a/config_app/c_app.py b/config_app/c_app.py index 049aef619..ea118d3b5 100644 --- a/config_app/c_app.py +++ b/config_app/c_app.py @@ -3,12 +3,13 @@ import logging from flask import Flask -from data import database +from data import database, model from util.config.superusermanager import SuperUserManager from util.ipresolver import NoopIPResolver from config_app._init_config import ROOT_DIR from config_app.config_util.config import get_config_provider +from util.security.instancekeys import InstanceKeys app = Flask(__name__) @@ -35,3 +36,6 @@ else: config_provider.update_app_config(app.config) superusers = SuperUserManager(app) ip_resolver = NoopIPResolver() +instance_keys = InstanceKeys(app) + +model.config.app_config = app.config diff --git a/config_app/config_endpoints/api/__init__.py b/config_app/config_endpoints/api/__init__.py index d539639eb..0adf080be 100644 --- a/config_app/config_endpoints/api/__init__.py +++ b/config_app/config_endpoints/api/__init__.py @@ -137,4 +137,5 @@ import config_endpoints.api.discovery import config_endpoints.api.suconfig import config_endpoints.api.superuser import config_endpoints.api.user +import config_endpoints.api.tar_config_loader diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index 539a6599c..ec4a84105 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -7,7 +7,7 @@ from flask import abort, request from config_app.config_endpoints.api.suconfig_models_pre_oci import pre_oci_model as model from config_app.config_endpoints.api import resource, ApiResource, nickname, validate_json_request -from config_app.c_app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver +from config_app.c_app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver, instance_keys from auth.auth_context import get_authenticated_user from data.users import get_federated_service_name, get_users_handler @@ -15,16 +15,13 @@ from data.database import configure from data.runmigration import run_alembic_migration from util.config.configutil import add_enterprise_config_defaults from util.config.database import sync_database_with_config -from util.config.validator import validate_service_for_config, ValidatorContext +from util.config.validator import validate_service_for_config, ValidatorContext, is_valid_config_upload_filename logger = logging.getLogger(__name__) def database_is_valid(): """ Returns whether the database, as configured, is valid. """ - if app.config['TESTING']: - return False - return model.is_valid() @@ -103,9 +100,6 @@ class SuperUserConfig(ApiResource): # Link the existing user to the external user. model.attach_federated_login(current_user.username, service_name, result.username) - # Ensure database is up-to-date with config - sync_database_with_config(config_object) - return { 'exists': True, 'config': config_object @@ -182,11 +176,12 @@ class SuperUserSetupDatabase(ApiResource): configure(combined) app.config['DB_URI'] = combined['DB_URI'] + db_uri = app.config['DB_URI'] log_handler = _AlembicLogHandler() try: - run_alembic_migration(log_handler) + run_alembic_migration(db_uri, log_handler, setup_app=False) except Exception as ex: return { 'error': str(ex) @@ -301,9 +296,42 @@ class SuperUserConfigValidate(ApiResource): if not config_provider.config_exists(): config = request.get_json()['config'] validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''), - ip_resolver=ip_resolver, - config_provider=config_provider) + instance_keys=instance_keys, + ip_resolver=ip_resolver, + config_provider=config_provider) return validate_service_for_config(service, validator_context) abort(403) + + +@resource('/v1/superuser/config/file/') +class SuperUserConfigFile(ApiResource): + """ Resource for fetching the status of config files and overriding them. """ + @nickname('scConfigFileExists') + def get(self, filename): + """ Returns whether the configuration file with the given name exists. """ + if not is_valid_config_upload_filename(filename): + abort(404) + + return { + 'exists': config_provider.volume_file_exists(filename) + } + + + @nickname('scUpdateConfigFile') + def post(self, filename): + """ Updates the configuration file with the given name. """ + if not is_valid_config_upload_filename(filename): + abort(404) + + # Note: This method can be called before the configuration exists + # to upload the database SSL cert. + uploaded_file = request.files['file'] + if not uploaded_file: + abort(400) + + config_provider.save_volume_file(filename, uploaded_file) + return { + 'status': True + } diff --git a/config_app/config_endpoints/api/suconfig_models_pre_oci.py b/config_app/config_endpoints/api/suconfig_models_pre_oci.py index df83b8e9f..655b0c1da 100644 --- a/config_app/config_endpoints/api/suconfig_models_pre_oci.py +++ b/config_app/config_endpoints/api/suconfig_models_pre_oci.py @@ -4,6 +4,8 @@ from config_app.config_endpoints.api.suconfig_models_interface import SuperuserC class PreOCIModel(SuperuserConfigDataInterface): + # Note: this method is different than has_users: the user select will throw if the user + # table does not exist, whereas has_users assumes the table is valid def is_valid(self): try: list(User.select().limit(1)) diff --git a/config_app/config_endpoints/api/superuser.py b/config_app/config_endpoints/api/superuser.py index c061adacf..0a4705d03 100644 --- a/config_app/config_endpoints/api/superuser.py +++ b/config_app/config_endpoints/api/superuser.py @@ -1,6 +1,7 @@ -import os import logging import pathvalidate +import os + from flask import request, jsonify from config_app.config_endpoints.exception import InvalidRequest @@ -46,13 +47,14 @@ class SuperUserCustomCertificate(ApiResource): logger.exception('Got IO error for cert %s', certpath) return '', 204 + # TODO(QUAY-991): properly install the custom certs provided by user # Call the update script to install the certificate immediately. - if not app.config['TESTING']: - logger.debug('Calling certs_install.sh') - if os.system('/conf/init/certs_install.sh') != 0: - raise Exception('Could not install certificates') - - logger.debug('certs_install.sh completed') + # if not app.config['TESTING']: + # logger.debug('Calling certs_install.sh') + # if os.system('/conf/init/certs_install.sh') != 0: + # raise Exception('Could not install certificates') + # + # logger.debug('certs_install.sh completed') return '', 204 diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py new file mode 100644 index 000000000..63b57d214 --- /dev/null +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -0,0 +1,28 @@ +import tarfile + +from flask import request, make_response + +from data.database import configure + +from config_app.c_app import app, config_provider +from config_app.config_endpoints.api import resource, ApiResource, nickname + +@resource('/v1/configapp/tarconfig') +class TarConfigLoader(ApiResource): + """ Resource for validating a block of configuration against an external service. """ + + @nickname('uploadTarballConfig') + def put(self): + """ Loads tarball config into the config provider """ + input_stream = request.stream + tar_stream = tarfile.open(mode="r|gz", fileobj=input_stream) + + config_provider.load_from_tar_stream(tar_stream) + + # now try to connect to the db provided in their config + combined = dict(**app.config) + combined.update(config_provider.get_config()) + + configure(combined) + + return make_response('OK') diff --git a/config_app/config_util/config/__init__.py b/config_app/config_util/config/__init__.py index d32c159d8..c344bb415 100644 --- a/config_app/config_util/config/__init__.py +++ b/config_app/config_util/config/__init__.py @@ -1,12 +1,12 @@ from config_app.config_util.config.fileprovider import FileConfigProvider from config_app.config_util.config.testprovider import TestConfigProvider +from config_app.config_util.config.inmemoryprovider import InMemoryProvider def get_config_provider(config_volume, yaml_filename, py_filename, testing=False): """ Loads and returns the config provider for the current environment. """ + if testing: return TestConfigProvider() - return FileConfigProvider(config_volume, yaml_filename, py_filename) - - + return InMemoryProvider() diff --git a/config_app/config_util/config/inmemoryprovider.py b/config_app/config_util/config/inmemoryprovider.py new file mode 100644 index 000000000..7db5d1a89 --- /dev/null +++ b/config_app/config_util/config/inmemoryprovider.py @@ -0,0 +1,84 @@ +import logging +import yaml +import io +import os + +from config_app.config_util.config.baseprovider import BaseProvider + + +logger = logging.getLogger(__name__) + +CONFIG_FILENAME = 'config.yaml' + +class InMemoryProvider(BaseProvider): + def __init__(self): + self.files = {} + self.config = {} + self.was_loaded = False + + @property + def provider_id(self): + return 'memory' + + def update_app_config(self, app_config): + self.config = app_config + + def get_config(self): + return self.config + + def save_config(self, config_object): + self.config = config_object + self.was_loaded = True + + def config_exists(self): + return self.was_loaded + + def volume_exists(self): + return True + + def volume_file_exists(self, filename): + return any([name.startswith(filename) for name in self.files]) + + def get_volume_file(self, filename, mode='r'): + return io.BytesIO(self.files[filename]) + + def write_volume_file(self, filename, contents): + raise Exception('Not implemented yet') + + def remove_volume_file(self, filename): + raise Exception('Not implemented yet') + + def list_volume_directory(self, path): + def strip_directory(string): + if '/' in string: + return string[string.rfind('/') + 1:] + return string + + return [strip_directory(name) for name in self.files if name.startswith(path)] + + def save_volume_file(self, filename, flask_file): + self.files[filename] = flask_file.read() + + def requires_restart(self, app_config): + raise Exception('Not implemented yet') + + def get_volume_path(self, directory, filename): + return os.path.join(directory, filename) + + def load_from_tarball(self, tarfile): + for tarinfo in tarfile.getmembers(): + if tarinfo.isfile(): + self.files[tarinfo.name] = tarfile.extractfile(tarinfo.name).read() + + if self.files.has_key(CONFIG_FILENAME): + self.config = yaml.load(self.files.get(CONFIG_FILENAME)) + self.was_loaded = True + + def load_from_tar_stream(self, tarfile): + for tarinfo in tarfile: + if tarinfo.isfile(): + self.files[tarinfo.name] = tarfile.extractfile(tarinfo).read() + + if self.files.has_key(CONFIG_FILENAME): + self.config = yaml.load(self.files.get(CONFIG_FILENAME)) + self.was_loaded = True diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.html b/config_app/js/components/config-setup-app/config-setup-app.component.html index 52430bc35..d1ea58f31 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.html +++ b/config_app/js/components/config-setup-app/config-setup-app.component.html @@ -23,4 +23,4 @@
- + diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.ts b/config_app/js/components/config-setup-app/config-setup-app.component.ts index 6b002e1db..e22ca57c0 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.ts +++ b/config_app/js/components/config-setup-app/config-setup-app.component.ts @@ -22,4 +22,8 @@ export class ConfigSetupAppComponent { private chooseLoad(): void { this.state = 'load'; } + + private configLoaded(): void { + this.state = 'setup'; + } } diff --git a/config_app/js/components/cor-option/cor-option.html b/config_app/js/components/cor-option/cor-option.html new file mode 100644 index 000000000..8482a9050 --- /dev/null +++ b/config_app/js/components/cor-option/cor-option.html @@ -0,0 +1,3 @@ +
  • + +
    • diff --git a/config_app/js/components/cor-option/cor-option.js b/config_app/js/components/cor-option/cor-option.js new file mode 100644 index 000000000..880d83df3 --- /dev/null +++ b/config_app/js/components/cor-option/cor-option.js @@ -0,0 +1,32 @@ +const corOption = require('./cor-option.html'); +const corOptionsMenu = require('./cor-options-menu.html'); + +angular.module('quay-config') + .directive('corOptionsMenu', function() { + var directiveDefinitionObject = { + priority: 1, + templateUrl: corOptionsMenu, + replace: true, + transclude: true, + restrict: 'C', + scope: {}, + controller: function($rootScope, $scope, $element) { + } + }; + return directiveDefinitionObject; + }) + .directive('corOption', function() { + var directiveDefinitionObject = { + priority: 1, + templateUrl: corOption, + replace: true, + transclude: true, + restrict: 'C', + scope: { + 'optionClick': '&optionClick' + }, + controller: function($rootScope, $scope, $element) { + } + }; + return directiveDefinitionObject; + }); diff --git a/config_app/js/components/cor-option/cor-options-menu.html b/config_app/js/components/cor-option/cor-options-menu.html new file mode 100644 index 000000000..a234590a3 --- /dev/null +++ b/config_app/js/components/cor-option/cor-options-menu.html @@ -0,0 +1,6 @@ + +
    + +
      +
      +       diff --git a/config_app/js/components/cor-title/cor-title-content.html b/config_app/js/components/cor-title/cor-title-content.html new file mode 100644 index 000000000..0d3e13ddd --- /dev/null +++ b/config_app/js/components/cor-title/cor-title-content.html @@ -0,0 +1,3 @@ +
      +

      +
      diff --git a/config_app/js/components/cor-title/cor-title.html b/config_app/js/components/cor-title/cor-title.html new file mode 100644 index 000000000..63cfd322c --- /dev/null +++ b/config_app/js/components/cor-title/cor-title.html @@ -0,0 +1,2 @@ +
      + diff --git a/config_app/js/components/cor-title/cor-title.js b/config_app/js/components/cor-title/cor-title.js new file mode 100644 index 000000000..033112f23 --- /dev/null +++ b/config_app/js/components/cor-title/cor-title.js @@ -0,0 +1,31 @@ + +const titleUrl = require('./cor-title.html'); +const titleContentUrl = require('./cor-title-content.html'); + +angular.module('quay-config') + .directive('corTitleContent', function() { + var directiveDefinitionObject = { + priority: 1, + templateUrl: titleContentUrl, + replace: true, + transclude: true, + restrict: 'C', + scope: {}, + controller: function($rootScope, $scope, $element) { + } + }; + return directiveDefinitionObject; + }) + .directive('corTitle', function() { + var directiveDefinitionObject = { + priority: 1, + templateUrl: titleUrl, + replace: true, + transclude: true, + restrict: 'C', + scope: {}, + controller: function($rootScope, $scope, $element) { + } + }; + return directiveDefinitionObject; + }); diff --git a/config_app/js/components/file-upload-box.js b/config_app/js/components/file-upload-box.js index 2a48b06d0..ae2f6ec3a 100644 --- a/config_app/js/components/file-upload-box.js +++ b/config_app/js/components/file-upload-box.js @@ -17,6 +17,7 @@ angular.module('quay-config').directive('fileUploadBox', function () { 'filesValidated': '&filesValidated', 'extensions': ' exists. + */ +angular.module('quay-config').directive("filesChanged", [function () { + return { + restrict: 'A', + scope: { + 'filesChanged': "&" + }, + link: function (scope, element, attributes) { + element.bind("change", function (changeEvent) { + scope.$apply(function() { + scope.filesChanged({'files': changeEvent.target.files}); + }); + }); + } + } +}]); diff --git a/config_app/js/components/load-config/load-config.component.ts b/config_app/js/components/load-config/load-config.component.ts index f8a3ba47a..eb93b36cd 100644 --- a/config_app/js/components/load-config/load-config.component.ts +++ b/config_app/js/components/load-config/load-config.component.ts @@ -1,11 +1,58 @@ -import { Input, Component, Inject } from 'ng-metadata/core'; +import {Component, EventEmitter, Inject, Output} from 'ng-metadata/core'; const templateUrl = require('./load-config.html'); +const styleUrl = require('./load-config.css'); + +declare var bootbox: any; @Component({ selector: 'load-config', templateUrl, + styleUrls: [ styleUrl ], }) export class LoadConfigComponent { - constructor() { + private readyToSubmit: boolean = false; + private uploadFunc: Function; + @Output() public configLoaded: EventEmitter = new EventEmitter(); + + private constructor(@Inject('ApiService') private apiService: any) { + } + + private handleTarballSelected(files: File[], callback: Function) { + this.readyToSubmit = true; + callback(true) + } + + private handleTarballCleared() { + this.readyToSubmit = false; + } + + private uploadTarball() { + this.uploadFunc(success => { + if (success) { + this.configLoaded.emit({}); + } else { + bootbox.dialog({ + "message": 'Could not upload configuration. Please check you have provided a valid tar file' + + 'If this problem persists, please contact support', + "title": 'Error Loading Configuration', + "buttons": { + "close": { + "label": "Close", + "className": "btn-primary" + } + } + }); + } + }); + } + + /** + * When files are validated, this is called by the child to give us + * the callback function to upload + * @param files: files to upload + * @param uploadFiles: function to call to upload files + */ + private tarballValidatedByUploadBox(files, uploadFiles) { + this.uploadFunc = uploadFiles; } } \ No newline at end of file diff --git a/config_app/js/components/load-config/load-config.css b/config_app/js/components/load-config/load-config.css new file mode 100644 index 000000000..e69de29bb diff --git a/config_app/js/components/load-config/load-config.html b/config_app/js/components/load-config/load-config.html index 60f78ff2b..1a4d101e6 100644 --- a/config_app/js/components/load-config/load-config.html +++ b/config_app/js/components/load-config/load-config.html @@ -8,9 +8,20 @@

      Load Config

      -
      - Please upload a tarball -
      +
      + Please upload the previous configuration +
      +
      +
      +
      diff --git a/config_app/js/config-field-templates/config-certificates-field.html b/config_app/js/config-field-templates/config-certificates-field.html index f20e4c459..f2316871f 100644 --- a/config_app/js/config-field-templates/config-certificates-field.html +++ b/config_app/js/config-field-templates/config-certificates-field.html @@ -20,10 +20,11 @@ Upload certificates:
      + api-endpoint="superuser/customcerts" + select-message="Select custom certificate to add to configuration. Must be in PEM format and end extension '.crt'" + files-selected="handleCertsSelected(files, callback)" + reset="resetUpload" + extensions="['.crt']"> diff --git a/config_app/js/config-field-templates/config-setup-tool.html b/config_app/js/config-field-templates/config-setup-tool.html index 629e3b45f..ad86d3107 100644 --- a/config_app/js/config-field-templates/config-setup-tool.html +++ b/config_app/js/config-field-templates/config-setup-tool.html @@ -16,7 +16,7 @@
      - Basic Configuration + Basic Configuration
      @@ -456,7 +456,7 @@
      - BitTorrent-based download + BitTorrent-based download
      @@ -941,7 +941,7 @@
      - GitHub (Enterprise) Authentication + GitHub (Enterprise) Authentication
      @@ -1049,7 +1049,7 @@
      - Google Authentication + Google Authentication
      @@ -1390,7 +1390,7 @@
      - GitHub (Enterprise) Build Triggers + GitHub (Enterprise) Build Triggers
      diff --git a/config_app/js/core-config-setup/config-setup-tool.html b/config_app/js/core-config-setup/config-setup-tool.html index ec3faa1c7..9bec737fa 100644 --- a/config_app/js/core-config-setup/config-setup-tool.html +++ b/config_app/js/core-config-setup/config-setup-tool.html @@ -17,7 +17,7 @@
      - Basic Configuration + Basic Configuration
      @@ -457,7 +457,7 @@
      - BitTorrent-based download + BitTorrent-based download
      @@ -942,7 +942,7 @@
      - GitHub (Enterprise) Authentication + GitHub (Enterprise) Authentication
      @@ -1050,7 +1050,7 @@
      - Google Authentication + Google Authentication
      @@ -1391,7 +1391,7 @@
      - GitHub (Enterprise) Build Triggers + GitHub (Enterprise) Build Triggers
      diff --git a/config_app/js/core-config-setup/core-config-setup.js b/config_app/js/core-config-setup/core-config-setup.js index d7f91d9bf..30126e43b 100644 --- a/config_app/js/core-config-setup/core-config-setup.js +++ b/config_app/js/core-config-setup/core-config-setup.js @@ -7,6 +7,7 @@ const urlListField = require('../config-field-templates/config-list-field.html') const urlFileField = require('../config-field-templates/config-file-field.html'); const urlBoolField = require('../config-field-templates/config-bool-field.html'); const urlNumericField = require('../config-field-templates/config-numeric-field.html'); +const urlContactField = require('../config-field-templates/config-contact-field.html'); const urlContactsField = require('../config-field-templates/config-contacts-field.html'); const urlMapField = require('../config-field-templates/config-map-field.html'); const urlServiceKeyField = require('../config-field-templates/config-service-key-field.html'); @@ -1115,7 +1116,7 @@ angular.module("quay-config") .directive('configContactField', function () { var directiveDefinitionObject = { priority: 0, - templateUrl: urlContactsField, + templateUrl: urlContactField, priority: 1, replace: false, transclude: true, @@ -1413,11 +1414,7 @@ angular.module("quay-config") }); }; - // UserService.updateUserIn($scope, function(user) { - // console.log(user) - // no need to check for user, since it's all local loadCertificates(); - // }); $scope.handleCertsSelected = function(files, callback) { $scope.certsUploading = true; diff --git a/config_app/js/setup/setup.component.js b/config_app/js/setup/setup.component.js index 3e828214c..b8fe72bd3 100644 --- a/config_app/js/setup/setup.component.js +++ b/config_app/js/setup/setup.component.js @@ -65,9 +65,6 @@ const templateUrl = require('./setup.html'); // Database is being setup. 'DB_SETUP': 'setup-db', - // Database setup has succeeded. - 'DB_SETUP_SUCCESS': 'setup-db-success', - // An error occurred when setting up the database. 'DB_SETUP_ERROR': 'setup-db-error', @@ -262,7 +259,6 @@ const templateUrl = require('./setup.html'); $scope.createSuperUser = function() { $scope.currentStep = $scope.States.CREATING_SUPERUSER; ApiService.scCreateInitialSuperuser($scope.superUser, null).then(function(resp) { - UserService.load(); $scope.checkStatus(); }, function(resp) { $scope.currentStep = $scope.States.SUPERUSER_ERROR; @@ -277,7 +273,7 @@ const templateUrl = require('./setup.html'); $scope.currentStep = $scope.States.DB_SETUP_ERROR; $scope.errors.DatabaseSetupError = resp['error']; } else { - $scope.currentStep = $scope.States.DB_SETUP_SUCCESS; + $scope.currentStep = $scope.States.CREATE_SUPERUSER; } }, ApiService.errorDisplay('Could not setup database. Please report this to support.')) }; diff --git a/config_app/js/setup/setup.html b/config_app/js/setup/setup.html index bfbc194dc..93cc87a87 100644 --- a/config_app/js/setup/setup.html +++ b/config_app/js/setup/setup.html @@ -1,4 +1,4 @@ -
      +
      @@ -12,11 +12,11 @@ - + - + @@ -24,8 +24,8 @@
      Configure your Redis database and other settings below
      - +
      @@ -39,11 +39,11 @@ - + - +

      Setup

      @@ -95,7 +95,7 @@

      - + is currently being restarted

      This can take several minutes. If the container does not restart on its own, diff --git a/config_app/static/css/cor-option.css b/config_app/static/css/cor-option.css new file mode 100644 index 000000000..97ae7887d --- /dev/null +++ b/config_app/static/css/cor-option.css @@ -0,0 +1,8 @@ +.cor-options-menu .fa-cog { + color: #999; + cursor: pointer; +} + +.open .fa-cog { + color: #428BCA; +} diff --git a/config_app/static/css/cor-title.css b/config_app/static/css/cor-title.css new file mode 100644 index 000000000..ef199785a --- /dev/null +++ b/config_app/static/css/cor-title.css @@ -0,0 +1,4 @@ +.cor-title { + display: flex; + justify-content: center; +} diff --git a/config_app/static/img/network-tile.png b/config_app/static/img/network-tile.png new file mode 100644 index 000000000..c27deaff2 Binary files /dev/null and b/config_app/static/img/network-tile.png differ diff --git a/config_app/static/img/redis-small.png b/config_app/static/img/redis-small.png new file mode 100644 index 000000000..c330a720e Binary files /dev/null and b/config_app/static/img/redis-small.png differ diff --git a/config_app/static/img/rocket.png b/config_app/static/img/rocket.png new file mode 100644 index 000000000..b9ffddf39 Binary files /dev/null and b/config_app/static/img/rocket.png differ diff --git a/data/migrations/env.py b/data/migrations/env.py index 10296c1b1..d920dd13d 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -14,14 +14,23 @@ from peewee import SqliteDatabase from data.database import all_models, db from data.migrations.tester import NoopTester, PopulateTestDataTester -from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata from release import GIT_HEAD, REGION, SERVICE from util.morecollections import AttrDict config = context.config -config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI'])) +DB_URI = config.get_main_option('db_uri', 'sqlite:///test/data/test.db') + +# This option exists because alembic needs the db proxy to be configured in order +# to perform migrations. The app import does the init of the proxy, but we don't +# want that in the case of the config app, as we are explicitly connecting to a +# db that the user has passed in, and we can't have import dependency on app +if config.get_main_option('alembic_setup_app', 'True') == 'True': + from app import app + DB_URI = app.config['DB_URI'] + +config.set_main_option('sqlalchemy.url', unquote(DB_URI)) # Interpret the config file for Python logging. # This line sets up loggers basically. if config.config_file_name: @@ -47,7 +56,7 @@ def get_tester(): connecting to a production database. """ if os.environ.get('TEST_MIGRATE', '') == 'true': - url = unquote(app.config['DB_URI']) + url = unquote(DB_URI) if url.find('.quay.io') < 0: return PopulateTestDataTester() @@ -65,13 +74,12 @@ def run_migrations_offline(): script output. """ - url = unquote(app.config['DB_URI']) + url = unquote(DB_URI) context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True) with context.begin_transaction(): context.run_migrations(tables=tables, tester=get_tester()) - def run_migrations_online(): """Run migrations in 'online' mode. diff --git a/data/runmigration.py b/data/runmigration.py index b06cf861d..f4126aba1 100644 --- a/data/runmigration.py +++ b/data/runmigration.py @@ -5,12 +5,19 @@ from alembic.script import ScriptDirectory from alembic.environment import EnvironmentContext from alembic.migration import __name__ as migration_name -def run_alembic_migration(log_handler=None): +def run_alembic_migration(db_uri, log_handler=None, setup_app=True): if log_handler: logging.getLogger(migration_name).addHandler(log_handler) config = Config() config.set_main_option("script_location", "data:migrations") + config.set_main_option("db_uri", db_uri) + + if setup_app: + config.set_main_option('alembic_setup_app', 'True') + else: + config.set_main_option('alembic_setup_app', '') + script = ScriptDirectory.from_config(config) def fn(rev, context): diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 2ed481219..77954c6ae 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -118,7 +118,7 @@ class SuperUserSetupDatabase(ApiResource): log_handler = _AlembicLogHandler() try: - run_alembic_migration(log_handler) + run_alembic_migration(app.config['DB_URI'], log_handler) except Exception as ex: return { 'error': str(ex)