Fix alembic migrations importing app

Ensure we connect to loaded config db

config_app/c_app.py

@@ -3,7 +3,7 @@ import logging
 
 from flask import Flask
 
-from data import database
+from data import database, model
 from util.config.superusermanager import SuperUserManager
 from util.ipresolver import NoopIPResolver
 
@@ -35,3 +35,5 @@ else:
 config_provider.update_app_config(app.config)
 superusers = SuperUserManager(app)
 ip_resolver = NoopIPResolver()
+
+model.config.app_config = app.config

config_app/config_endpoints/api/suconfig.py

@@ -22,9 +22,6 @@ logger = logging.getLogger(__name__)
 
 def database_is_valid():
     """ Returns whether the database, as configured, is valid. """
-    if app.config['TESTING']:
-        return False
-
     return model.is_valid()
 
 
@@ -103,9 +100,6 @@ class SuperUserConfig(ApiResource):
                     # Link the existing user to the external user.
                     model.attach_federated_login(current_user.username, service_name, result.username)
 
-            # Ensure database is up-to-date with config
-            sync_database_with_config(config_object)
-
             return {
                 'exists': True,
                 'config': config_object
@@ -182,11 +176,12 @@ class SuperUserSetupDatabase(ApiResource):
 
             configure(combined)
             app.config['DB_URI'] = combined['DB_URI']
+            db_uri = app.config['DB_URI']
 
             log_handler = _AlembicLogHandler()
 
             try:
-                run_alembic_migration(log_handler)
+                run_alembic_migration(db_uri, log_handler)
             except Exception as ex:
                 return {
                     'error': str(ex)

config_app/config_endpoints/api/suconfig_models_pre_oci.py

@@ -4,6 +4,8 @@ from config_app.config_endpoints.api.suconfig_models_interface import SuperuserC
 
 
 class PreOCIModel(SuperuserConfigDataInterface):
+    # Note: this method is different than has_users: the user select will throw if the user
+    # table does not exist, whereas has_users assumes the table is valid
     def is_valid(self):
         try:
             list(User.select().limit(1))

config_app/config_endpoints/api/superuser.py

@@ -79,9 +79,8 @@ class SuperUserCustomCertificates(ApiResource):
         cert_views = []
         for extra_cert_path in extra_certs_found:
             try:
-                cert_full_path = config_provider.get_volume_path(EXTRA_CA_DIRECTORY, extra_cert_path)
+                cert = config_provider.get_volume_path(EXTRA_CA_DIRECTORY, extra_cert_path)
-                with config_provider.get_volume_file(cert_full_path) as f:
+                certificate = load_certificate(cert)
-                    certificate = load_certificate(f.read())
                 cert_views.append({
                     'path': extra_cert_path,
                     'names': list(certificate.names),

config_app/config_endpoints/api/tar_config_loader.py

@@ -4,7 +4,9 @@ import cStringIO
 
 from flask import request, make_response
 
-from config_app.c_app import config_provider
+from data.database import configure
+
+from config_app.c_app import app, config_provider
 from config_app.config_endpoints.api import resource, ApiResource, nickname, validate_json_request
 
 logger = logging.getLogger(__name__)
@@ -25,4 +27,10 @@ class TarConfigLoader(ApiResource):
         # TODO(sam): refactor config provider to accept a stream write to avoid loading into memory
         config_provider.load_from_tarball(config)
 
+        # now try to connect to the db provided in their config
+        combined = dict(**app.config)
+        combined.update(config_provider.get_config())
+
+        configure(combined)
+
         return make_response('OK')

config_app/config_util/config/__init__.py

@@ -5,12 +5,12 @@ from config_app.config_util.config.inmemoryprovider import InMemoryProvider
 
 def get_config_provider(config_volume, yaml_filename, py_filename, testing=False):
     """ Loads and returns the config provider for the current environment. """
-    if True:
-        return InMemoryProvider()
 
     if testing:
         return TestConfigProvider()
 
-    return FileConfigProvider(config_volume, yaml_filename, py_filename)
+    else:
+        return InMemoryProvider()
+
 
 

config_app/config_util/config/inmemoryprovider.py

@@ -12,6 +12,7 @@ class InMemoryProvider(BaseProvider):
     def __init__(self):
         self.files = {}
         self.config = {}
+        self.was_loaded = False
 
     @property
     def provider_id(self):
@@ -24,16 +25,17 @@ class InMemoryProvider(BaseProvider):
         return self.config
 
     def save_config(self, config_object):
-        raise Exception('Not implemented yet')
+        self.config = config_object
+        self.was_loaded = True
 
     def config_exists(self):
-        raise Exception('Not implemented yet')
+        return self.was_loaded
 
     def volume_exists(self):
-        raise Exception('Not implemented yet')
+        return True
 
     def volume_file_exists(self, filename):
-        return filename in self.files
+        return any([ name.startswith(filename) for name in self.files ])
 
     def get_volume_file(self, filename, mode='r'):
         return self.files[filename]
@@ -45,7 +47,7 @@ class InMemoryProvider(BaseProvider):
         raise Exception('Not implemented yet')
 
     def list_volume_directory(self, path):
-        return [ name for name in self.files ]
+        return [ name for name in self.files if name.startswith(path) ]
 
     def save_volume_file(self, filename, flask_file):
         raise Exception('Not implemented yet')
@@ -54,7 +56,8 @@ class InMemoryProvider(BaseProvider):
         raise Exception('Not implemented yet')
 
     def get_volume_path(self, directory, filename):
-        raise Exception('Not implemented yet')
+        # Here we can just access the filename since we're storing the tarball files with their full path
+        return self.files[filename]
 
     def load_from_tarball(self, tarfile):
         for tarinfo in tarfile.getmembers():
@@ -63,4 +66,5 @@ class InMemoryProvider(BaseProvider):
                     self.config = yaml.load(tarfile.extractfile(tarinfo.name).read())
                 else:
                     self.files[tarinfo.name] = tarfile.extractfile(tarinfo.name).read()
+        self.was_loaded = True
 

config_app/js/components/config-setup-app/config-setup-app.component.html

@@ -23,4 +23,4 @@
     </div>
 </div>
 <div ng-if="$ctrl.state === 'setup'" class="setup"></div>
-<load-config ng-if="$ctrl.state === 'load'"></load-config>
+<load-config ng-if="$ctrl.state === 'load'" config-loaded="$ctrl.configLoaded()"></load-config>

config_app/js/components/config-setup-app/config-setup-app.component.ts

@@ -22,4 +22,8 @@ export class ConfigSetupAppComponent {
     private chooseLoad(): void {
         this.state = 'load';
     }
+
+    private configLoaded(): void {
+        this.state = 'setup';
+    }
 }

config_app/js/components/load-config/load-config.component.ts

@@ -1,4 +1,4 @@
-import { Component, Inject } from 'ng-metadata/core';
+import {Component, EventEmitter, Inject, Output} from 'ng-metadata/core';
 const templateUrl = require('./load-config.html');
 const styleUrl = require('./load-config.css');
 
@@ -10,7 +10,7 @@ const styleUrl = require('./load-config.css');
 export class LoadConfigComponent {
     private readyToSubmit: boolean = false;
     private uploadFunc: Function;
-    private state: 'load' | 'validate' = 'load';
+    @Output() public configLoaded: EventEmitter<any> = new EventEmitter();
 
     private constructor(@Inject('ApiService') private apiService: any) {
     }
@@ -27,9 +27,8 @@ export class LoadConfigComponent {
     private uploadTarball() {
         this.uploadFunc(success => {
             if (success) {
-                this.state = 'validate';
+                this.configLoaded.emit({});
-            }
+            } else {
-            else {
                 this.apiService.errorDisplay('Error loading configuration',
                     'Could not upload configuration. Please reload the page and try again.\n' +
                     'If this problem persists, please contact support')();

config_app/js/components/load-config/load-config.css

@@ -1,4 +0,0 @@
-
-
-.load-config__body strong {
-}

config_app/js/components/load-config/load-config.html

@@ -1,4 +1,4 @@
-<div ng-if="$ctrl.state === 'load'">
+<div>
     <div class="co-dialog modal fade initial-setup-modal in" id="setupModal" style="display: block;">
         <div class="modal-backdrop fade in" style="height: 1000px;"></div>
         <div class="modal-dialog fade in">
@@ -26,22 +26,3 @@
         </div><!-- /.modal-dialog -->
     </div>
 </div>
-
-<div ng-if="$ctrl.state === 'validate'">
-    <div class="co-dialog modal fade initial-setup-modal in" id="validateModal" style="display: block;">
-        <div class="modal-backdrop fade in" style="height: 1000px;"></div>
-        <div class="modal-dialog fade in">
-            <div class="modal-content">
-                <!-- Header -->
-                <div class="modal-header">
-                    <h4 class="modal-title"><span>Validate Config</span></h4>
-                </div>
-                <!-- Body -->
-                <div class="modal-body">
-                    <span>Validating Config...</span>
-                    spinner here...
-                </div>
-            </div><!-- /.modal-content -->
-        </div><!-- /.modal-dialog -->
-    </div>
-</div>

config_app/js/core-config-setup/core-config-setup.js

@@ -190,9 +190,10 @@ angular.module("quay-config")
         };
 
         $scope.validateHostname = function(hostname) {
-          if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
+            // TODO(sam): maybe revert?
-            return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
+          // if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
-          }
+          //   return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
+          // }
 
           return null;
         };

config_app/js/setup/setup.component.js

@@ -31,9 +31,10 @@ const templateUrl = require('./setup.html');
         $scope.HOSTNAME_REGEX = '^[a-zA-Z-0-9_\.\-]+(:[0-9]+)?$';
 
         $scope.validateHostname = function(hostname) {
-            if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
+            // TODO(sam): maybe revert?
-                return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
+            // if (hostname.indexOf('127.0.0.1') == 0 || hostname.indexOf('localhost') == 0) {
-            }
+            //     return 'Please specify a non-localhost hostname. "localhost" will refer to the container, not your machine.'
+            // }
 
             return null;
         };
@@ -65,9 +66,6 @@ const templateUrl = require('./setup.html');
             // Database is being setup.
             'DB_SETUP': 'setup-db',
 
-            // Database setup has succeeded.
-            'DB_SETUP_SUCCESS': 'setup-db-success',
-
             // An error occurred when setting up the database.
             'DB_SETUP_ERROR': 'setup-db-error',
 
@@ -262,7 +260,7 @@ const templateUrl = require('./setup.html');
         $scope.createSuperUser = function() {
             $scope.currentStep = $scope.States.CREATING_SUPERUSER;
             ApiService.scCreateInitialSuperuser($scope.superUser, null).then(function(resp) {
-                UserService.load();
+                // UserService.load();
                 $scope.checkStatus();
             }, function(resp) {
                 $scope.currentStep = $scope.States.SUPERUSER_ERROR;
@@ -277,7 +275,7 @@ const templateUrl = require('./setup.html');
                     $scope.currentStep = $scope.States.DB_SETUP_ERROR;
                     $scope.errors.DatabaseSetupError = resp['error'];
                 } else {
-                    $scope.currentStep = $scope.States.DB_SETUP_SUCCESS;
+                    $scope.currentStep = $scope.States.CREATE_SUPERUSER;
                 }
             }, ApiService.errorDisplay('Could not setup database. Please report this to support.'))
         };

config_app/js/setup/setup.html

@@ -24,8 +24,8 @@
         <div>Configure your Redis database and other settings below</div>
       </div>
 
-      <config-setup-tool is-active="isStep(currentStep, States.CONFIG)"
+      <div class="config-setup-tool" is-active="isStep(currentStep, States.CONFIG)"
-          configuration-saved="configurationSaved(config)"></config-setup-tool>
+          configuration-saved="configurationSaved(config)"></divconfig-setup-tool>
     </div>
   </div>
 </div>
@@ -39,7 +39,7 @@
         <span class="cor-step-bar" progress="stepProgress">
           <span class="cor-step" title="Configure Database" text="1"></span>
           <span class="cor-step" title="Setup Database" icon="database"></span>
-          <span class="cor-step" title="Container Restart" icon="refresh"></span>
+          <span class="cor-step" title="Container Restart" icon="sync"></span>
           <span class="cor-step" title="Create Superuser" text="2"></span>
           <span class="cor-step" title="Configure Registry" text="3"></span>
           <span class="cor-step" title="Validate Configuration" text="4"></span>
@@ -95,7 +95,7 @@
       <div class="modal-body" style="padding: 20px;"
            ng-show="isStep(currentStep, States.DB_RESTARTING, States.CONFIG_RESTARTING)">
            <h4 style="margin-bottom: 20px;">
-            <i class="fa fa-lg fa-refresh" style="margin-right: 10px;"></i>
+            <i class="fa fa-lg fa-sync" style="margin-right: 10px;"></i>
              <span class="registry-name"></span> is currently being restarted
            </h4>
            This can take several minutes. If the container does not restart on its own,

data/migrations/env.py

@@ -14,14 +14,13 @@ from peewee import SqliteDatabase
 from data.database import all_models, db
 from data.migrations.tester import NoopTester, PopulateTestDataTester
 
-from app import app
 from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
 from release import GIT_HEAD, REGION, SERVICE
 from util.morecollections import AttrDict
 
 config = context.config
-config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI']))
+DB_URI = config.get_main_option('db_uri')
-
+config.set_main_option('sqlalchemy.url', unquote(DB_URI))
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 if config.config_file_name:
@@ -47,7 +46,7 @@ def get_tester():
         connecting to a production database.
     """
     if os.environ.get('TEST_MIGRATE', '') == 'true':
-        url = unquote(app.config['DB_URI'])
+        url = unquote(DB_URI)
         if url.find('.quay.io') < 0:
             return PopulateTestDataTester()
 
@@ -65,12 +64,11 @@ def run_migrations_offline():
     script output.
 
     """
-    url = unquote(app.config['DB_URI'])
+    url = unquote(DB_URI)
     context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True)
 
     with context.begin_transaction():
-        context.run_migrations(tables=tables, tester=get_tester())
+        context.run_migrations(tables=tables)
-
 
 def run_migrations_online():
     """Run migrations in 'online' mode.
@@ -99,7 +97,7 @@ def run_migrations_online():
     try:
         with context.begin_transaction():
             try:
-                context.run_migrations(tables=tables, tester=get_tester())
+                context.run_migrations(tables=tables)
             except (CommandError, ResolutionError) as ex:
                 if 'No such revision' not in str(ex):
                     raise

data/runmigration.py

@@ -5,12 +5,13 @@ from alembic.script import ScriptDirectory
 from alembic.environment import EnvironmentContext
 from alembic.migration import __name__ as migration_name
 
-def run_alembic_migration(log_handler=None):
+def run_alembic_migration(db_uri, log_handler=None):
   if log_handler:
     logging.getLogger(migration_name).addHandler(log_handler)
 
   config = Config()
   config.set_main_option("script_location", "data:migrations")
+  config.set_main_option("db_uri", db_uri)
   script = ScriptDirectory.from_config(config)
 
   def fn(rev, context):

endpoints/api/suconfig.py

@@ -118,7 +118,7 @@ class SuperUserSetupDatabase(ApiResource):
       log_handler = _AlembicLogHandler()
 
       try:
-        run_alembic_migration(log_handler)
+        run_alembic_migration(app.config['DB_URI'], log_handler)
       except Exception as ex:
         return {
           'error': str(ex)