vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix alembic migrations importing app

Browse source 
Ensure we connect to loaded config db
This commit is contained in:
Sam Chow 2018-06-19 13:46:34 -04:00
parent bb2b28cd11
commit b5f630ba29
18 changed files with 69 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
config_app/c_app.py
View file

@ -3,7 +3,7 @@ import logging
from flask import Flask
from data import database
from data import database, model
from util.config.superusermanager import SuperUserManager
from util.ipresolver import NoopIPResolver
@ -35,3 +35,5 @@ else:
config_provider.update_app_config(app.config)
superusers = SuperUserManager(app)
ip_resolver = NoopIPResolver()
model.config.app_config = app.config

9
config_app/config_endpoints/api/suconfig.py
View file

@ -22,9 +22,6 @@ logger = logging.getLogger(__name__)
def database_is_valid():
""" Returns whether the database, as configured, is valid. """
if app.config['TESTING']:
return False
return model.is_valid()
@ -103,9 +100,6 @@ class SuperUserConfig(ApiResource):
# Link the existing user to the external user.
model.attach_federated_login(current_user.username, service_name, result.username)
# Ensure database is up-to-date with config
sync_database_with_config(config_object)
return {
'exists': True,
'config': config_object
@ -182,11 +176,12 @@ class SuperUserSetupDatabase(ApiResource):
configure(combined)
app.config['DB_URI'] = combined['DB_URI']
db_uri = app.config['DB_URI']
log_handler = _AlembicLogHandler()
try:
run_alembic_migration(log_handler)
run_alembic_migration(db_uri, log_handler)
except Exception as ex:
return {
'error': str(ex)

2
config_app/config_endpoints/api/suconfig_models_pre_oci.py
View file

@ -4,6 +4,8 @@ from config_app.config_endpoints.api.suconfig_models_interface import SuperuserC
class PreOCIModel(SuperuserConfigDataInterface):
# Note: this method is different than has_users: the user select will throw if the user
# table does not exist, whereas has_users assumes the table is valid
def is_valid(self):
try:
list(User.select().limit(1))

15
config_app/config_endpoints/api/superuser.py
View file

@ -79,14 +79,13 @@ class SuperUserCustomCertificates(ApiResource):
cert_views = []
for extra_cert_path in extra_certs_found:
try:
cert_full_path = config_provider.get_volume_path(EXTRA_CA_DIRECTORY, extra_cert_path)
with config_provider.get_volume_file(cert_full_path) as f:
certificate = load_certificate(f.read())
cert_views.append({
'path': extra_cert_path,
'names': list(certificate.names),
'expired': certificate.expired,
})
cert = config_provider.get_volume_path(EXTRA_CA_DIRECTORY, extra_cert_path)
certificate = load_certificate(cert)
cert_views.append({
'path': extra_cert_path,
'names': list(certificate.names),
'expired': certificate.expired,
})
except CertInvalidException as cie:
cert_views.append({
'path': extra_cert_path,

10
config_app/config_endpoints/api/tar_config_loader.py
View file

@ -4,7 +4,9 @@ import cStringIO
from flask import request, make_response
from config_app.c_app import config_provider
from data.database import configure
from config_app.c_app import app, config_provider
from config_app.config_endpoints.api import resource, ApiResource, nickname, validate_json_request
logger = logging.getLogger(__name__)
@ -25,4 +27,10 @@ class TarConfigLoader(ApiResource):
# TODO(sam): refactor config provider to accept a stream write to avoid loading into memory
config_provider.load_from_tarball(config)
# now try to connect to the db provided in their config
combined = dict(**app.config)
combined.update(config_provider.get_config())
configure(combined)
return make_response('OK')

6
config_app/config_util/config/__init__.py
View file

@ -5,12 +5,12 @@ from config_app.config_util.config.inmemoryprovider import InMemoryProvider
def get_config_provider(config_volume, yaml_filename, py_filename, testing=False):
""" Loads and returns the config provider for the current environment. """
if True:
return InMemoryProvider()
if testing:
return TestConfigProvider()
return FileConfigProvider(config_volume, yaml_filename, py_filename)
else:
return InMemoryProvider()

16
config_app/config_util/config/inmemoryprovider.py
View file

@ -12,6 +12,7 @@ class InMemoryProvider(BaseProvider):
def __init__(self):
self.files = {}
self.config = {}
self.was_loaded = False
@property
def provider_id(self):
@ -24,16 +25,17 @@ class InMemoryProvider(BaseProvider):
return self.config
def save_config(self, config_object):
raise Exception('Not implemented yet')
self.config = config_object
self.was_loaded = True
def config_exists(self):
raise Exception('Not implemented yet')
return self.was_loaded
def volume_exists(self):
raise Exception('Not implemented yet')
return True
def volume_file_exists(self, filename):
return filename in self.files
return any([ name.startswith(filename) for name in self.files ])
def get_volume_file(self, filename, mode='r'):
return self.files[filename]
@ -45,7 +47,7 @@ class InMemoryProvider(BaseProvider):
raise Exception('Not implemented yet')
def list_volume_directory(self, path):
return [ name for name in self.files ]
return [ name for name in self.files if name.startswith(path) ]
def save_volume_file(self, filename, flask_file):
raise Exception('Not implemented yet')
@ -54,7 +56,8 @@ class InMemoryProvider(BaseProvider):
raise Exception('Not implemented yet')
def get_volume_path(self, directory, filename):
raise Exception('Not implemented yet')
# Here we can just access the filename since we're storing the tarball files with their full path
return self.files[filename]
def load_from_tarball(self, tarfile):
for tarinfo in tarfile.getmembers():
@ -63,4 +66,5 @@ class InMemoryProvider(BaseProvider):
self.config = yaml.load(tarfile.extractfile(tarinfo.name).read())
else:
self.files[tarinfo.name] = tarfile.extractfile(tarinfo.name).read()
self.was_loaded = True

2
config_app/js/components/config-setup-app/config-setup-app.component.html
View file

@ -23,4 +23,4 @@
</div>
</div>
<div ng-if="$ctrl.state === 'setup'" class="setup"></div>
<load-config ng-if="$ctrl.state === 'load'"></load-config>
<load-config ng-if="$ctrl.state === 'load'" config-loaded="$ctrl.configLoaded()"></load-config>

4
config_app/js/components/config-setup-app/config-setup-app.component.ts
View file

@ -22,4 +22,8 @@ export class ConfigSetupAppComponent {
private chooseLoad(): void {
this.state = 'load';
}
private configLoaded(): void {
this.state = 'setup';
}
}

9
config_app/js/components/load-config/load-config.component.ts
View file

@ -1,4 +1,4 @@
import { Component, Inject } from 'ng-metadata/core';
import {Component, EventEmitter, Inject, Output} from 'ng-metadata/core';
const templateUrl = require('./load-config.html');
const styleUrl = require('./load-config.css');
@ -10,7 +10,7 @@ const styleUrl = require('./load-config.css');
export class LoadConfigComponent {
private readyToSubmit: boolean = false;
private uploadFunc: Function;
private state: 'load' | 'validate' = 'load';
@Output() public configLoaded: EventEmitter<any> = new EventEmitter();
private constructor(@Inject('ApiService') private apiService: any) {
}
@ -27,9 +27,8 @@ export class LoadConfigComponent {
private uploadTarball() {
this.uploadFunc(success => {
if (success) {
this.state = 'validate';
}
else {
this.configLoaded.emit({});
} else {
this.apiService.errorDisplay('Error loading configuration',
'Could not upload configuration. Please reload the page and try again.\n' +
'If this problem persists, please contact support')();

4
config_app/js/components/load-config/load-config.css
View file

@ -1,4 +0,0 @@
.load-config__body strong {
}

21
config_app/js/components/load-config/load-config.html
View file

@ -1,4 +1,4 @@
<div ng-if="$ctrl.state === 'load'">
<div>
<div class="co-dialog modal fade initial-setup-modal in" id="setupModal" style="display: block;">
<div class="modal-backdrop fade in" style="height: 1000px;"></div>
<div class="modal-dialog fade in">
@ -26,22 +26,3 @@
</div><!-- /.modal-dialog -->
</div>
</div>
<div ng-if="$ctrl.state === 'validate'">
<div class="co-dialog modal fade initial-setup-modal in" id="validateModal" style="display: block;">
<div class="modal-backdrop fade in" style="height: 1000px;"></div>
<div class="modal-dialog fade in">
<div class="modal-content">
<!-- Header -->
<div class="modal-header">
<h4 class="modal-title"><span>Validate Config</span></h4>
</div>
<!-- Body -->
<div class="modal-body">
<span>Validating Config...</span>
spinner here...
</div>
</div><!-- /.modal-content -->
</div><!-- /.modal-dialog -->
</div>
</div>

7
config_app/js/core-config-setup/core-config-setup.js
View file

@ -190,9 +190,10 @@ angular.module("quay-config")
};
$scope.validateHostname = function(hostname) {
if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
}
// TODO(sam): maybe revert?
// if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
// return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
// }
return null;
};

14
config_app/js/setup/setup.component.js
View file

@ -31,9 +31,10 @@ const templateUrl = require('./setup.html');
$scope.HOSTNAME_REGEX = '^[a-zA-Z-0-9_\.\-]+(:[0-9]+)?$';
$scope.validateHostname = function(hostname) {
if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
}
// TODO(sam): maybe revert?
// if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
// return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
// }
return null;
};
@ -65,9 +66,6 @@ const templateUrl = require('./setup.html');
// Database is being setup.
'DB_SETUP': 'setup-db',
// Database setup has succeeded.
'DB_SETUP_SUCCESS': 'setup-db-success',
// An error occurred when setting up the database.
'DB_SETUP_ERROR': 'setup-db-error',
@ -262,7 +260,7 @@ const templateUrl = require('./setup.html');
$scope.createSuperUser = function() {
$scope.currentStep = $scope.States.CREATING_SUPERUSER;
ApiService.scCreateInitialSuperuser($scope.superUser, null).then(function(resp) {
UserService.load();
// UserService.load();
$scope.checkStatus();
}, function(resp) {
$scope.currentStep = $scope.States.SUPERUSER_ERROR;
@ -277,7 +275,7 @@ const templateUrl = require('./setup.html');
$scope.currentStep = $scope.States.DB_SETUP_ERROR;
$scope.errors.DatabaseSetupError = resp['error'];
} else {
$scope.currentStep = $scope.States.DB_SETUP_SUCCESS;
$scope.currentStep = $scope.States.CREATE_SUPERUSER;
}
}, ApiService.errorDisplay('Could not setup database. Please report this to support.'))
};

8
config_app/js/setup/setup.html
View file

@ -24,8 +24,8 @@
<div>Configure your Redis database and other settings below</div>
</div>
<config-setup-tool is-active="isStep(currentStep, States.CONFIG)"
configuration-saved="configurationSaved(config)"></config-setup-tool>
<div class="config-setup-tool" is-active="isStep(currentStep, States.CONFIG)"
configuration-saved="configurationSaved(config)"></divconfig-setup-tool>
</div>
</div>
</div>
@ -39,7 +39,7 @@
<span class="cor-step-bar" progress="stepProgress">
<span class="cor-step" title="Configure Database" text="1"></span>
<span class="cor-step" title="Setup Database" icon="database"></span>
<span class="cor-step" title="Container Restart" icon="refresh"></span>
<span class="cor-step" title="Container Restart" icon="sync"></span>
<span class="cor-step" title="Create Superuser" text="2"></span>
<span class="cor-step" title="Configure Registry" text="3"></span>
<span class="cor-step" title="Validate Configuration" text="4"></span>
@ -95,7 +95,7 @@
<div class="modal-body" style="padding: 20px;"
ng-show="isStep(currentStep, States.DB_RESTARTING, States.CONFIG_RESTARTING)">
<h4 style="margin-bottom: 20px;">
<i class="fa fa-lg fa-refresh" style="margin-right: 10px;"></i>
<i class="fa fa-lg fa-sync" style="margin-right: 10px;"></i>
<span class="registry-name"></span> is currently being restarted
</h4>
This can take several minutes. If the container does not restart on its own,

14
data/migrations/env.py
View file

@ -14,14 +14,13 @@ from peewee import SqliteDatabase
from data.database import all_models, db
from data.migrations.tester import NoopTester, PopulateTestDataTester
from app import app
from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
from release import GIT_HEAD, REGION, SERVICE
from util.morecollections import AttrDict
config = context.config
config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI']))
DB_URI = config.get_main_option('db_uri')
config.set_main_option('sqlalchemy.url', unquote(DB_URI))
# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name:
@ -47,7 +46,7 @@ def get_tester():
connecting to a production database.
"""
if os.environ.get('TEST_MIGRATE', '') == 'true':
url = unquote(app.config['DB_URI'])
url = unquote(DB_URI)
if url.find('.quay.io') < 0:
return PopulateTestDataTester()
@ -65,12 +64,11 @@ def run_migrations_offline():
script output.
"""
url = unquote(app.config['DB_URI'])
url = unquote(DB_URI)
context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True)
with context.begin_transaction():
context.run_migrations(tables=tables, tester=get_tester())
context.run_migrations(tables=tables)
def run_migrations_online():
"""Run migrations in 'online' mode.
@ -99,7 +97,7 @@ def run_migrations_online():
try:
with context.begin_transaction():
try:
context.run_migrations(tables=tables, tester=get_tester())
context.run_migrations(tables=tables)
except (CommandError, ResolutionError) as ex:
if 'No such revision' not in str(ex):
raise

3
data/runmigration.py
View file

@ -5,12 +5,13 @@ from alembic.script import ScriptDirectory
from alembic.environment import EnvironmentContext
from alembic.migration import __name__ as migration_name
def run_alembic_migration(log_handler=None):
def run_alembic_migration(db_uri, log_handler=None):
if log_handler:
logging.getLogger(migration_name).addHandler(log_handler)
config = Config()
config.set_main_option("script_location", "data:migrations")
config.set_main_option("db_uri", db_uri)
script = ScriptDirectory.from_config(config)
def fn(rev, context):

2
endpoints/api/suconfig.py
View file

@ -118,7 +118,7 @@ class SuperUserSetupDatabase(ApiResource):
log_handler = _AlembicLogHandler()
try:
run_alembic_migration(log_handler)
run_alembic_migration(app.config['DB_URI'], log_handler)
except Exception as ex:
return {
'error': str(ex)