From 17fc72d2625ca7b178bf3cbc3b9c4a42f13ba4a2 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 18 Nov 2014 14:07:33 -0500 Subject: [PATCH] Switch postgres to a non-transactional DDL to allow us to use peewee to modify data in migrations: enterprise customers are running postgres migrations offline already. Move the image backfill script back to a migration since it will now work. Unify the interface to sending a DB URI to env.py for the migration script. --- conf/init/runmigration.sh | 5 +---- data/migrations/env.py | 21 +++--------------- data/migrations/migration.sh | 22 ++++++++++--------- ...5_calculate_uncompressed_sizes_for_all_.py | 4 +--- ...translate_the_queue_names_to_reference_.py | 4 ++-- util/uncompressedsize.py | 10 ++------- 6 files changed, 21 insertions(+), 45 deletions(-) diff --git a/conf/init/runmigration.sh b/conf/init/runmigration.sh index a82349dcf..8b006b745 100755 --- a/conf/init/runmigration.sh +++ b/conf/init/runmigration.sh @@ -2,7 +2,4 @@ set -e # Run the database migration -PYTHONPATH=. venv/bin/alembic upgrade head - -# Run the uncompressed size migration -PYTHONPATH=. venv/bin/python -m util.uncompressedsize \ No newline at end of file +PYTHONPATH=. venv/bin/alembic upgrade head \ No newline at end of file diff --git a/data/migrations/env.py b/data/migrations/env.py index 650683572..3b2df5186 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -13,24 +13,8 @@ from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata from util.morecollections import AttrDict -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -db_uri = unquote(app.config['DB_URI']) -if 'GENMIGRATE' in os.environ: - docker_host = os.environ.get('DOCKER_HOST') - docker_host_ip = docker_host[len('tcp://'):].split(':')[0] - if os.environ.get('GENMIGRATE') == 'mysql': - db_uri = 'mysql+pymysql://root:password@%s/genschema' % (docker_host_ip) - else: - db_uri = 'postgresql://postgres@%s/genschema' % (docker_host_ip) - -if 'DB_URI' in os.environ: - db_uri = os.environ['DB_URI'] - -app.config['DB_URI'] = db_uri - config = context.config -config.set_main_option('sqlalchemy.url', db_uri) +config.set_main_option('sqlalchemy.url', unquote(app.config['DB_URI'])) # Interpret the config file for Python logging. # This line sets up loggers basically. @@ -86,7 +70,8 @@ def run_migrations_online(): connection = engine.connect() context.configure( connection=connection, - target_metadata=target_metadata + target_metadata=target_metadata, + transactional_ddl=False, ) try: diff --git a/data/migrations/migration.sh b/data/migrations/migration.sh index 5deeba3dd..dedaf2445 100755 --- a/data/migrations/migration.sh +++ b/data/migrations/migration.sh @@ -1,4 +1,8 @@ -set -e +set -e + +DOCKER_IP=`echo $DOCKER_HOST | sed 's/tcp:\/\///' | sed 's/:.*//'` +MYSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"mysql+pymysql://root:password@$DOCKER_IP/genschema\"}" +PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\"}" up_mysql() { # Run a SQL database on port 3306 inside of Docker. @@ -36,19 +40,19 @@ down_postgres() { gen_migrate() { # Generate a SQLite database with the schema as defined by the existing alembic model. - GENMIGRATE=$1 PYTHONPATH=. alembic upgrade head + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head # Generate the migration to the current model. - GENMIGRATE=$1 PYTHONPATH=. alembic revision --autogenerate -m "$2" + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic revision --autogenerate -m "$2" } test_migrate() { # Generate a SQLite database with the schema as defined by the existing alembic model. - GENMIGRATE=$1 PYTHONPATH=. alembic upgrade head + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head # Downgrade to verify it works in both directions. COUNT=`ls data/migrations/versions/*.py | wc -l | tr -d ' '` - GENMIGRATE=$1 PYTHONPATH=. alembic downgrade "-$COUNT" + QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT" } # Test (and generate, if requested) via MySQL. @@ -59,13 +63,13 @@ if [ ! -z "$@" ] then set +e echo '> Generating Migration' - gen_migrate "mysql" "$@" + gen_migrate $MYSQL_CONFIG_OVERRIDE "$@" set -e fi echo '> Testing Migration (mysql)' set +e -test_migrate "mysql" +test_migrate $MYSQL_CONFIG_OVERRIDE set -e down_mysql @@ -75,8 +79,6 @@ up_postgres echo '> Testing Migration (postgres)' set +e -test_migrate "postgres" +test_migrate $PGSQL_CONFIG_OVERRIDE set -e down_postgres - - diff --git a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py index 03f6d7733..df2ed6f14 100644 --- a/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py +++ b/data/migrations/versions/2430f55c41d5_calculate_uncompressed_sizes_for_all_.py @@ -16,9 +16,7 @@ from util.uncompressedsize import backfill_sizes_from_data def upgrade(tables): - # Note: Doing non-alembic operations inside alembic can cause a deadlock. This call has been - # moved to runmigration.sh. - pass + backfill_sizes_from_data() def downgrade(tables): pass diff --git a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py index 15b740cbf..76e698471 100644 --- a/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py +++ b/data/migrations/versions/2fb36d4be80d_translate_the_queue_names_to_reference_.py @@ -1,14 +1,14 @@ """Translate the queue names to reference namespace by id, remove the namespace column. Revision ID: 2430f55c41d5 -Revises: 9a1087b007d +Revises: 313d297811c4 Create Date: 2014-09-30 17:31:33.308490 """ # revision identifiers, used by Alembic. revision = '2fb36d4be80d' -down_revision = '2430f55c41d5' +down_revision = '313d297811c4' from alembic import op import sqlalchemy as sa diff --git a/util/uncompressedsize.py b/util/uncompressedsize.py index 5a6f16602..d39c43540 100644 --- a/util/uncompressedsize.py +++ b/util/uncompressedsize.py @@ -1,9 +1,8 @@ import logging import zlib -import sys from data import model -from data.database import ImageStorage, configure +from data.database import ImageStorage from app import app, storage as store from data.database import db, db_random_func from util.gzipstream import ZLIB_GZIP_WINDOW @@ -15,16 +14,11 @@ logger = logging.getLogger(__name__) CHUNK_SIZE = 5 * 1024 * 1024 def backfill_sizes_from_data(): - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) logger.debug('Starting uncompressed image size backfill') logger.debug('NOTE: This can be a LONG RUNNING OPERATION. Please wait!') - # Make sure we have a reference to the current DB. - configure(app.config) - - logger.debug('Uncompressed backfill: Database configured') - # Check for any uncompressed images. has_images = bool(list(ImageStorage .select(ImageStorage.uuid)