From c78c4502117de635756c6ac6f8b108b3ff26410f Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 30 Oct 2015 15:40:24 -0400 Subject: [PATCH] UTF-8 v1_json_metadata, comment, manifest This will allow us to store unicode JSON blobs in the column on MySQL. --- ...da62_switch_manifest_text_to_a_longtext.py | 19 ++----------------- ..._migrate_image_data_back_to_image_table.py | 5 +++-- util/migrate/__init__.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py b/data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py index f11b5336a..bd139158c 100644 --- a/data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py +++ b/data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py @@ -15,28 +15,13 @@ import sqlalchemy as sa from sqlalchemy.types import TypeDecorator, Text from sqlalchemy.dialects.mysql import LONGTEXT -import uuid -class EngineLongText(TypeDecorator): - """Platform-independent LongText type. - - Uses MySQL's LONGTEXT type, otherwise uses - Text, because other engines are not as limited - as MySQL. - - """ - impl = Text - - def load_dialect_impl(self, dialect): - if dialect.name == 'mysql': - return dialect.type_descriptor(LONGTEXT()) - else: - return dialect.type_descriptor(Text()) +from util.migrate import UTF8LongText def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.drop_column(u'tagmanifest', 'json_data') - op.add_column(u'tagmanifest', sa.Column('json_data', EngineLongText(), nullable=False)) + op.add_column(u'tagmanifest', sa.Column('json_data', UTF8LongText(), nullable=False)) ### end Alembic commands ### diff --git a/data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py b/data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py index 17af21eb3..270968aae 100644 --- a/data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py +++ b/data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py @@ -13,14 +13,15 @@ down_revision = '3a3bb77e17d5' from alembic import op import sqlalchemy as sa +from util.migrate import UTF8LongText def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.add_column('image', sa.Column('aggregate_size', sa.BigInteger(), nullable=True)) op.add_column('image', sa.Column('command', sa.Text(), nullable=True)) - op.add_column('image', sa.Column('comment', sa.Text(), nullable=True)) + op.add_column('image', sa.Column('comment', UTF8LongText(), nullable=True)) op.add_column('image', sa.Column('created', sa.DateTime(), nullable=True)) - op.add_column('image', sa.Column('v1_json_metadata', sa.Text(), nullable=True)) + op.add_column('image', sa.Column('v1_json_metadata', UTF8LongText(), nullable=True)) ### end Alembic commands ### diff --git a/util/migrate/__init__.py b/util/migrate/__init__.py index e69de29bb..6b0a65feb 100644 --- a/util/migrate/__init__.py +++ b/util/migrate/__init__.py @@ -0,0 +1,16 @@ +from sqlalchemy.types import TypeDecorator, Text +from sqlalchemy.dialects.mysql import TEXT as MySQLText, LONGTEXT + +class UTF8LongText(TypeDecorator): + """ Platform-independent UTF-8 LONGTEXT type. + + Uses MySQL's LongText with charset utf8mb4, otherwise uses TEXT, because + other engines default to UTF-8 and have longer TEXT fields. + """ + impl = Text + + def load_dialect_impl(self, dialect): + if dialect.name == 'mysql': + return dialect.type_descriptor(LONGTEXT(charset='utf8mb4', collation='utf8mb4_unicode_ci')) + else: + return dialect.type_descriptor(Text())