From fc7756a3c2dce524439ea3c8d9b2949c8f1811db Mon Sep 17 00:00:00 2001 From: jakedt Date: Wed, 9 Apr 2014 19:11:33 -0400 Subject: [PATCH 1/2] Add alembic plumbing for database schema migrations. --- alembic.ini | 58 ++++++++++++++++++++++++++ config.py | 3 +- data/database.py | 31 +++++++++++--- data/migrations/env.py | 76 ++++++++++++++++++++++++++++++++++ data/migrations/script.py.mako | 22 ++++++++++ data/model/sqlalchemybridge.py | 76 ++++++++++++++++++++++++++++++++++ 6 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 alembic.ini create mode 100644 data/migrations/env.py create mode 100644 data/migrations/script.py.mako create mode 100644 data/model/sqlalchemybridge.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 000000000..5089cce9c --- /dev/null +++ b/alembic.ini @@ -0,0 +1,58 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = data/migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +sqlalchemy.url = sqlite:///will/be/overridden + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/config.py b/config.py index 4cb3a5d6b..09f883cb6 100644 --- a/config.py +++ b/config.py @@ -70,12 +70,11 @@ class DefaultConfig(object): TESTING = True # DB config - DB_NAME = 'test/data/test.db' + DB_URI = 'sqlite:///test/data/test.db' DB_CONNECTION_ARGS = { 'threadlocals': True, 'autorollback': True, } - DB_DRIVER_NAME = "SqliteDatabase" @staticmethod def create_transaction(db): diff --git a/data/database.py b/data/database.py index 07862e058..be0557637 100644 --- a/data/database.py +++ b/data/database.py @@ -5,19 +5,39 @@ import uuid from random import SystemRandom from datetime import datetime from peewee import * +from urlparse import urlparse from app import app logger = logging.getLogger(__name__) -DRIVER_LIST = { - 'SqliteDatabase': SqliteDatabase, - 'MySQLDatabase': MySQLDatabase, + +SCHEME_DRIVERS = { + 'mysql': MySQLDatabase, + 'sqlite': SqliteDatabase, } -db = DRIVER_LIST[app.config['DB_DRIVER_NAME']](app.config['DB_NAME'], - **app.config['DB_CONNECTION_ARGS']) + +def generate_db(config_object): + db_kwargs = dict(config_object['DB_CONNECTION_ARGS']) + connection_string = config_object['DB_URI'] + + scheme, auth_and_host, dbname = urlparse(connection_string)[:3] + if auth_and_host: + if '@' in auth_and_host: + auth, db_kwargs['host'] = auth_and_host.split('@') + if ':' in auth: + db_kwargs['user'], db_kwargs['passwd'] = auth.split(':') + + if scheme == 'sqlite': + dbname = dbname[1:] + + return SCHEME_DRIVERS[scheme](dbname, **db_kwargs) + + +db = generate_db(app.config) + def random_string_generator(length=16): def random_string(): @@ -46,6 +66,7 @@ class User(BaseModel): organization = BooleanField(default=False, index=True) robot = BooleanField(default=False, index=True) invoice_email = BooleanField(default=False) + new_user_field = CharField(null=True) class TeamRole(BaseModel): diff --git a/data/migrations/env.py b/data/migrations/env.py new file mode 100644 index 000000000..65f00819f --- /dev/null +++ b/data/migrations/env.py @@ -0,0 +1,76 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +from data.database import all_models +from app import app +from data.model.sqlalchemybridge import gen_sqlalchemy_metadata + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +config.set_main_option('sqlalchemy.url', app.config['DB_URI']) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = gen_sqlalchemy_metadata(all_models) + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = app.config['DB_CONNECTION'] + context.configure(url=url, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() + diff --git a/data/migrations/script.py.mako b/data/migrations/script.py.mako new file mode 100644 index 000000000..95702017e --- /dev/null +++ b/data/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/data/model/sqlalchemybridge.py b/data/model/sqlalchemybridge.py new file mode 100644 index 000000000..46809fb21 --- /dev/null +++ b/data/model/sqlalchemybridge.py @@ -0,0 +1,76 @@ +from sqlalchemy import (Table, MetaData, Column, ForeignKey, Integer, String, Boolean, Text, + DateTime, BigInteger, Index) +from peewee import (PrimaryKeyField, CharField, BooleanField, DateTimeField, TextField, + ForeignKeyField, BigIntegerField, IntegerField) + + +OPTIONS_TO_COPY = [ + 'null', + 'default', + 'primary_key', +] + + +OPTION_TRANSLATIONS = { + 'null': 'nullable', +} + + +def gen_sqlalchemy_metadata(peewee_model_list): + metadata = MetaData() + + for model in peewee_model_list: + meta = model._meta + + all_indexes = set(meta.indexes) + + columns = [] + for field in meta.get_fields(): + alchemy_type = None + col_args = [] + col_kwargs = {} + if isinstance(field, PrimaryKeyField): + alchemy_type = Integer + elif isinstance(field, CharField): + alchemy_type = String(field.max_length) + elif isinstance(field, BooleanField): + alchemy_type = Boolean + elif isinstance(field, DateTimeField): + alchemy_type = DateTime + elif isinstance(field, TextField): + alchemy_type = Text + elif isinstance(field, ForeignKeyField): + alchemy_type = Integer + target_name = '%s.%s' % (field.to_field.model_class._meta.db_table, + field.to_field.db_column) + col_args.append(ForeignKey(target_name)) + all_indexes.add(((field.name, ), field.unique)) + elif isinstance(field, BigIntegerField): + alchemy_type = BigInteger + elif isinstance(field, IntegerField): + alchemy_type = Integer + else: + raise RuntimeError('Unknown column type: %s' % field) + + for option_name in OPTIONS_TO_COPY: + alchemy_option_name = (OPTION_TRANSLATIONS[option_name] + if option_name in OPTION_TRANSLATIONS else option_name) + if alchemy_option_name not in col_kwargs: + option_val = getattr(field, option_name) + col_kwargs[alchemy_option_name] = option_val + + if field.unique or field.index: + all_indexes.add(((field.name, ), field.unique)) + + new_col = Column(field.db_column, alchemy_type, *col_args, **col_kwargs) + columns.append(new_col) + + new_table = Table(meta.db_table, metadata, *columns) + + for col_prop_names, unique in all_indexes: + col_names = [meta.fields[prop_name].db_column for prop_name in col_prop_names] + index_name = '%s_%s' % (meta.db_table, '_'.join(col_names)) + col_refs = [getattr(new_table.c, col_name) for col_name in col_names] + Index(index_name, *col_refs, unique=unique) + + return metadata From 4f3fa3420670f0c9bc25c8dfe3254b25130f9c2e Mon Sep 17 00:00:00 2001 From: jakedt Date: Wed, 9 Apr 2014 19:13:46 -0400 Subject: [PATCH 2/2] Remove test field from the database definition. --- data/database.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data/database.py b/data/database.py index be0557637..219222b0d 100644 --- a/data/database.py +++ b/data/database.py @@ -66,7 +66,6 @@ class User(BaseModel): organization = BooleanField(default=False, index=True) robot = BooleanField(default=False, index=True) invoice_email = BooleanField(default=False) - new_user_field = CharField(null=True) class TeamRole(BaseModel):