diff --git a/data/migrations/__init__.py b/data/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/data/migrations/env.py b/data/migrations/env.py index 04da8a142..10296c1b1 100644 --- a/data/migrations/env.py +++ b/data/migrations/env.py @@ -12,6 +12,8 @@ from urllib import unquote, quote from peewee import SqliteDatabase from data.database import all_models, db +from data.migrations.tester import NoopTester, PopulateTestDataTester + from app import app from data.model.sqlalchemybridge import gen_sqlalchemy_metadata from release import GIT_HEAD, REGION, SERVICE @@ -39,6 +41,18 @@ tables = AttrDict(target_metadata.tables) # my_important_option = config.get_main_option("my_important_option") # ... etc. +def get_tester(): + """ Returns the tester to use. We only return the tester that populates data + if the TEST_MIGRATE env var is set to `true` AND we make sure we're not + connecting to a production database. + """ + if os.environ.get('TEST_MIGRATE', '') == 'true': + url = unquote(app.config['DB_URI']) + if url.find('.quay.io') < 0: + return PopulateTestDataTester() + + return NoopTester() + def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -55,7 +69,8 @@ def run_migrations_offline(): context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True) with context.begin_transaction(): - context.run_migrations(tables=tables) + context.run_migrations(tables=tables, tester=get_tester()) + def run_migrations_online(): """Run migrations in 'online' mode. @@ -84,7 +99,7 @@ def run_migrations_online(): try: with context.begin_transaction(): try: - context.run_migrations(tables=tables) + context.run_migrations(tables=tables, tester=get_tester()) except (CommandError, ResolutionError) as ex: if 'No such revision' not in str(ex): raise diff --git a/data/migrations/migration.sh b/data/migrations/migration.sh index 8501090ef..a1d0f8f5d 100755 --- a/data/migrations/migration.sh +++ b/data/migrations/migration.sh @@ -8,7 +8,7 @@ PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\ up_mysql() { # Run a SQL database on port 3306 inside of Docker. - docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql + docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql:5.7 echo 'Sleeping for 25...' sleep 25 @@ -18,8 +18,8 @@ up_mysql() { } down_mysql() { - docker kill mysql - docker rm -v mysql + docker kill mysql || true + docker rm -v mysql || true } up_mariadb() { @@ -34,8 +34,8 @@ up_mariadb() { } down_mariadb() { - docker kill mariadb - docker rm -v mariadb + docker kill mariadb || true + docker rm -v mariadb || true } up_percona() { @@ -50,8 +50,8 @@ up_percona() { } down_percona() { - docker kill percona - docker rm -v percona + docker kill percona || true + docker rm -v percona || true } up_postgres() { @@ -67,8 +67,8 @@ up_postgres() { } down_postgres() { - docker kill postgres - docker rm -v postgres + docker kill postgres || true + docker rm -v postgres || true } gen_migrate() { @@ -83,14 +83,19 @@ gen_migrate() { test_migrate() { # Generate a database with the schema as defined by the existing alembic model. echo '> Running upgrade' - QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head + TEST_MIGRATE=true QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head # Downgrade to verify it works in both directions. echo '> Running downgrade' COUNT=`ls data/migrations/versions/*.py | wc -l | tr -d ' '` - QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT" + TEST_MIGRATE=true QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT" } +down_mysql +down_postgres +down_mariadb +down_percona + # Test (and generate, if requested) via MySQL. echo '> Starting MySQL' up_mysql diff --git a/data/migrations/script.py.mako b/data/migrations/script.py.mako index 1b92f9f48..3d82fbe98 100644 --- a/data/migrations/script.py.mako +++ b/data/migrations/script.py.mako @@ -14,9 +14,9 @@ from alembic import op import sqlalchemy as sa ${imports if imports else ""} -def upgrade(tables): +def upgrade(tables, tester): ${upgrades if upgrades else "pass"} -def downgrade(tables): +def downgrade(tables, tester): ${downgrades if downgrades else "pass"} diff --git a/data/migrations/tester.py b/data/migrations/tester.py new file mode 100644 index 000000000..762a47b56 --- /dev/null +++ b/data/migrations/tester.py @@ -0,0 +1,122 @@ +import json +import logging +import uuid + +from abc import ABCMeta, abstractmethod +from datetime import datetime +from six import add_metaclass + +from alembic import op +from sqlalchemy import text + +from util.abchelpers import nooper + +logger = logging.getLogger(__name__) + +def escape_table_name(table_name): + if op.get_bind().engine.name == 'postgresql': + # Needed for the `user` table. + return '"%s"' % table_name + + return table_name + + +class DataTypes(object): + @staticmethod + def DateTime(): + return datetime.now() + + @staticmethod + def Date(): + return datetime.now() + + @staticmethod + def String(): + return 'somestringvalue' + + @staticmethod + def UTF8Char(): + return 'some other value' + + @staticmethod + def UUID(): + return str(uuid.uuid4()) + + @staticmethod + def JSON(): + return json.dumps(dict(foo='bar', baz='meh')) + + @staticmethod + def Boolean(): + if op.get_bind().engine.name == 'postgresql': + return True + + return 1 + + @staticmethod + def BigInteger(): + return 21474836470 + + @staticmethod + def Integer(): + return 42 + + @staticmethod + def Foreign(table_name): + def get_index(): + result = op.get_bind().execute("SELECT id FROM %s LIMIT 1" % escape_table_name(table_name)) + try: + return list(result)[0][0] + except IndexError: + raise Exception('Could not find row for table %s' % table_name) + finally: + result.close() + + return get_index + + +@add_metaclass(ABCMeta) +class MigrationTester(object): + """ Implements an interface for adding testing capabilities to the + data model migration system in Alembic. + """ + TestDataType = DataTypes + + @abstractmethod + def populate_table(self, table_name, fields): + """ Called to populate a table with the given fields filled in with testing data. """ + + @abstractmethod + def populate_column(self, table_name, col_name, field_type): + """ Called to populate a column in a table to be filled in with testing data. """ + + +@nooper +class NoopTester(MigrationTester): + """ No-op version of the tester, designed for production workloads. """ + + +class PopulateTestDataTester(MigrationTester): + def populate_table(self, table_name, fields): + columns = {field_name: field_type() for field_name, field_type in fields} + field_name_vars = [':' + field_name for field_name, _ in fields] + + if op.get_bind().engine.name == 'postgresql': + field_names = ["%s" % field_name for field_name, _ in fields] + else: + field_names = ["`%s`" % field_name for field_name, _ in fields] + + table_name = escape_table_name(table_name) + query = text('INSERT INTO %s (%s) VALUES (%s)' % (table_name, ', '.join(field_names), + ', '.join(field_name_vars))) + logger.info("Executing test query %s with values %s", query, columns.values()) + op.get_bind().execute(query, **columns) + + def populate_column(self, table_name, col_name, field_type): + col_value = field_type() + row_id = DataTypes.Foreign(table_name)() + + table_name = escape_table_name(table_name) + update_text = text("UPDATE %s SET %s=:col_value where ID=:row_id" % (table_name, col_name)) + logger.info("Executing test query %s with value %s on row %s", update_text, col_value, row_id) + op.get_bind().execute(update_text, col_value=col_value, row_id=row_id) diff --git a/data/migrations/versions/0cf50323c78b_add_creation_date_to_user_table.py b/data/migrations/versions/0cf50323c78b_add_creation_date_to_user_table.py index 775c5caa7..cf8cb340a 100644 --- a/data/migrations/versions/0cf50323c78b_add_creation_date_to_user_table.py +++ b/data/migrations/versions/0cf50323c78b_add_creation_date_to_user_table.py @@ -14,13 +14,17 @@ from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.add_column('user', sa.Column('creation_date', sa.DateTime(), nullable=True)) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_column('user', 'creation_date', tester.TestDataType.DateTime) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'creation_date') # ### end Alembic commands ### diff --git a/data/migrations/versions/152bb29a1bb3_add_maximum_build_queue_count_setting_.py b/data/migrations/versions/152bb29a1bb3_add_maximum_build_queue_count_setting_.py index e1003ec8e..0ef47783e 100644 --- a/data/migrations/versions/152bb29a1bb3_add_maximum_build_queue_count_setting_.py +++ b/data/migrations/versions/152bb29a1bb3_add_maximum_build_queue_count_setting_.py @@ -14,13 +14,17 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.add_column('user', sa.Column('maximum_queued_builds_count', sa.Integer(), nullable=True)) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_column('user', 'maximum_queued_builds_count', tester.TestDataType.Integer) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'maximum_queued_builds_count') # ### end Alembic commands ### diff --git a/data/migrations/versions/152edccba18c_make_blodupload_byte_count_not_nullable.py b/data/migrations/versions/152edccba18c_make_blodupload_byte_count_not_nullable.py index cf66ef81d..474e796cb 100644 --- a/data/migrations/versions/152edccba18c_make_blodupload_byte_count_not_nullable.py +++ b/data/migrations/versions/152edccba18c_make_blodupload_byte_count_not_nullable.py @@ -14,11 +14,11 @@ from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(), nullable=False) -def downgrade(tables): +def downgrade(tables, tester): op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(), nullable=True) diff --git a/data/migrations/versions/1783530bee68_add_logentry2_table_quay_io_only.py b/data/migrations/versions/1783530bee68_add_logentry2_table_quay_io_only.py index 883e51899..32249b83a 100644 --- a/data/migrations/versions/1783530bee68_add_logentry2_table_quay_io_only.py +++ b/data/migrations/versions/1783530bee68_add_logentry2_table_quay_io_only.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.create_table('logentry2', sa.Column('id', sa.Integer(), nullable=False), @@ -40,7 +40,7 @@ def upgrade(tables): # ### end Alembic commands ### -def downgrade(tables): +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_table('logentry2') # ### end Alembic commands ### diff --git a/data/migrations/versions/17aff2e1354e_add_automatic_disable_of_build_triggers.py b/data/migrations/versions/17aff2e1354e_add_automatic_disable_of_build_triggers.py index b4c075b96..eb7212232 100644 --- a/data/migrations/versions/17aff2e1354e_add_automatic_disable_of_build_triggers.py +++ b/data/migrations/versions/17aff2e1354e_add_automatic_disable_of_build_triggers.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.add_column('repositorybuildtrigger', sa.Column('successive_failure_count', sa.Integer(), server_default='0', nullable=False)) op.add_column('repositorybuildtrigger', sa.Column('successive_internal_error_count', sa.Integer(), server_default='0', nullable=False)) @@ -28,7 +28,13 @@ def upgrade(tables): ], ) -def downgrade(tables): + # ### population of test data ### # + tester.populate_column('repositorybuildtrigger', 'successive_failure_count', tester.TestDataType.Integer) + tester.populate_column('repositorybuildtrigger', 'successive_internal_error_count', tester.TestDataType.Integer) + # ### end population of test data ### # + + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_column('repositorybuildtrigger', 'successive_internal_error_count') op.drop_column('repositorybuildtrigger', 'successive_failure_count') diff --git a/data/migrations/versions/224ce4c72c2f_add_last_accessed_field_to_user_table.py b/data/migrations/versions/224ce4c72c2f_add_last_accessed_field_to_user_table.py index 06326c566..41792b35b 100644 --- a/data/migrations/versions/224ce4c72c2f_add_last_accessed_field_to_user_table.py +++ b/data/migrations/versions/224ce4c72c2f_add_last_accessed_field_to_user_table.py @@ -14,14 +14,18 @@ from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.add_column('user', sa.Column('last_accessed', sa.DateTime(), nullable=True)) op.create_index('user_last_accessed', 'user', ['last_accessed'], unique=False) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_column('user', 'last_accessed', tester.TestDataType.DateTime) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_index('user_last_accessed', table_name='user') op.drop_column('user', 'last_accessed') diff --git a/data/migrations/versions/3e8cc74a1e7b_add_severity_and_media_type_to_global_.py b/data/migrations/versions/3e8cc74a1e7b_add_severity_and_media_type_to_global_.py index b88ff3875..93e90f945 100644 --- a/data/migrations/versions/3e8cc74a1e7b_add_severity_and_media_type_to_global_.py +++ b/data/migrations/versions/3e8cc74a1e7b_add_severity_and_media_type_to_global_.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.add_column('messages', sa.Column('media_type_id', sa.Integer(), nullable=False, server_default='1')) op.add_column('messages', sa.Column('severity', sa.String(length=255), nullable=False, server_default='info')) @@ -33,8 +33,14 @@ def upgrade(tables): {'name': 'text/markdown'}, ]) + # ### population of test data ### # + tester.populate_column('messages', 'media_type_id', tester.TestDataType.Foreign('mediatype')) + tester.populate_column('messages', 'severity', lambda: 'info') + tester.populate_column('messages', 'uuid', tester.TestDataType.UUID) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_constraint(op.f('fk_messages_media_type_id_mediatype'), 'messages', type_='foreignkey') op.drop_index('messages_uuid', table_name='messages') diff --git a/data/migrations/versions/45fd8b9869d4_add_notification_type.py b/data/migrations/versions/45fd8b9869d4_add_notification_type.py index b47e9cac9..e0d0a6842 100644 --- a/data/migrations/versions/45fd8b9869d4_add_notification_type.py +++ b/data/migrations/versions/45fd8b9869d4_add_notification_type.py @@ -12,14 +12,14 @@ down_revision = '94836b099894' from alembic import op -def upgrade(tables): +def upgrade(tables, tester): op.bulk_insert(tables.notificationkind, [ {'name': 'build_cancelled'}, ]) -def downgrade(tables): +def downgrade(tables, tester): op.execute(tables .notificationkind .delete() diff --git a/data/migrations/versions/53e2ac668296_remove_reference_to_subdir.py b/data/migrations/versions/53e2ac668296_remove_reference_to_subdir.py index fcfc518f4..be2bca4cf 100644 --- a/data/migrations/versions/53e2ac668296_remove_reference_to_subdir.py +++ b/data/migrations/versions/53e2ac668296_remove_reference_to_subdir.py @@ -33,11 +33,11 @@ def run_migration(migrate_function): log.warning("Failed to update build trigger %s with exception: ", trigger[0], e) -def upgrade(tables): +def upgrade(tables, tester): run_migration(delete_subdir) -def downgrade(tables): +def downgrade(tables, tester): run_migration(add_subdir) diff --git a/data/migrations/versions/5b7503aada1b_cleanup_old_robots.py b/data/migrations/versions/5b7503aada1b_cleanup_old_robots.py index d184790bc..a48a06d3c 100644 --- a/data/migrations/versions/5b7503aada1b_cleanup_old_robots.py +++ b/data/migrations/versions/5b7503aada1b_cleanup_old_robots.py @@ -15,9 +15,9 @@ import sqlalchemy as sa from util.migrate.cleanup_old_robots import cleanup_old_robots -def upgrade(tables): +def upgrade(tables, tester): cleanup_old_robots() -def downgrade(tables): +def downgrade(tables, tester): # Nothing to do. pass diff --git a/data/migrations/versions/61cadbacb9fc_add_ability_for_build_triggers_to_be_.py b/data/migrations/versions/61cadbacb9fc_add_ability_for_build_triggers_to_be_.py index fc03f48b4..22f808859 100644 --- a/data/migrations/versions/61cadbacb9fc_add_ability_for_build_triggers_to_be_.py +++ b/data/migrations/versions/61cadbacb9fc_add_ability_for_build_triggers_to_be_.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.create_table('disablereason', sa.Column('id', sa.Integer(), nullable=False), @@ -40,8 +40,13 @@ def upgrade(tables): op.create_foreign_key(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', 'disablereason', ['disabled_reason_id'], ['id']) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_column('repositorybuildtrigger', 'disabled_reason_id', tester.TestDataType.Foreign('disablereason')) + tester.populate_column('repositorybuildtrigger', 'enabled', tester.TestDataType.Boolean) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_constraint(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', type_='foreignkey') op.drop_index('repositorybuildtrigger_disabled_reason_id', table_name='repositorybuildtrigger') diff --git a/data/migrations/versions/6c7014e84a5e_add_user_prompt_support.py b/data/migrations/versions/6c7014e84a5e_add_user_prompt_support.py index 3959fafd0..80555de42 100644 --- a/data/migrations/versions/6c7014e84a5e_add_user_prompt_support.py +++ b/data/migrations/versions/6c7014e84a5e_add_user_prompt_support.py @@ -13,7 +13,7 @@ down_revision = 'c156deb8845d' from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.create_table('userpromptkind', sa.Column('id', sa.Integer(), nullable=False), @@ -39,8 +39,14 @@ def upgrade(tables): {'name':'confirm_username'}, ]) + # ### population of test data ### # + tester.populate_table('userprompt', [ + ('user_id', tester.TestDataType.Foreign('user')), + ('kind_id', tester.TestDataType.Foreign('userpromptkind')), + ]) + # ### end population of test data ### # -def downgrade(tables): +def downgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.drop_table('userprompt') op.drop_table('userpromptkind') diff --git a/data/migrations/versions/7367229b38d9_add_support_for_app_specific_tokens.py b/data/migrations/versions/7367229b38d9_add_support_for_app_specific_tokens.py index ffb0f00ef..8122b4a2e 100644 --- a/data/migrations/versions/7367229b38d9_add_support_for_app_specific_tokens.py +++ b/data/migrations/versions/7367229b38d9_add_support_for_app_specific_tokens.py @@ -15,7 +15,7 @@ import sqlalchemy as sa from sqlalchemy.dialects import mysql from util.migrate import UTF8CharField -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.create_table('appspecificauthtoken', sa.Column('id', sa.Integer(), nullable=False), @@ -40,7 +40,20 @@ def upgrade(tables): {'name': 'revoke_app_specific_token'}, ]) -def downgrade(tables): + # ### population of test data ### # + tester.populate_table('appspecificauthtoken', [ + ('user_id', tester.TestDataType.Foreign('user')), + ('uuid', tester.TestDataType.UUID), + ('title', tester.TestDataType.UTF8Char), + ('token_code', tester.TestDataType.String), + ('created', tester.TestDataType.DateTime), + ('expiration', tester.TestDataType.DateTime), + ('last_accessed', tester.TestDataType.DateTime), + ]) + # ### end population of test data ### # + + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_table('appspecificauthtoken') # ### end Alembic commands ### diff --git a/data/migrations/versions/7a525c68eb13_add_oci_app_models.py b/data/migrations/versions/7a525c68eb13_add_oci_app_models.py index 8ece0be27..1f97274d2 100644 --- a/data/migrations/versions/7a525c68eb13_add_oci_app_models.py +++ b/data/migrations/versions/7a525c68eb13_add_oci_app_models.py @@ -16,7 +16,7 @@ from sqlalchemy.dialects import mysql from util.migrate import UTF8LongText, UTF8CharField -def upgrade(tables): +def upgrade(tables, tester): op.create_table( 'tagkind', sa.Column('id', sa.Integer(), nullable=False), @@ -307,7 +307,7 @@ def upgrade(tables): ) -def downgrade(tables): +def downgrade(tables, tester): op.drop_table('manifestlayerscan') op.drop_table('manifestlayerdockerv1') op.drop_table('tag') diff --git a/data/migrations/versions/87fbbc224f10_add_disabled_datetime_to_trigger.py b/data/migrations/versions/87fbbc224f10_add_disabled_datetime_to_trigger.py index 5cb4df7ff..662474f76 100644 --- a/data/migrations/versions/87fbbc224f10_add_disabled_datetime_to_trigger.py +++ b/data/migrations/versions/87fbbc224f10_add_disabled_datetime_to_trigger.py @@ -14,14 +14,18 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.add_column('repositorybuildtrigger', sa.Column('disabled_datetime', sa.DateTime(), nullable=True)) op.create_index('repositorybuildtrigger_disabled_datetime', 'repositorybuildtrigger', ['disabled_datetime'], unique=False) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_column('repositorybuildtrigger', 'disabled_datetime', tester.TestDataType.DateTime) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_index('repositorybuildtrigger_disabled_datetime', table_name='repositorybuildtrigger') op.drop_column('repositorybuildtrigger', 'disabled_datetime') diff --git a/data/migrations/versions/94836b099894_create_new_notification_type.py b/data/migrations/versions/94836b099894_create_new_notification_type.py index ad8f147e9..7fe01de20 100644 --- a/data/migrations/versions/94836b099894_create_new_notification_type.py +++ b/data/migrations/versions/94836b099894_create_new_notification_type.py @@ -13,14 +13,14 @@ down_revision = 'faf752bd2e0a' from alembic import op -def upgrade(tables): +def upgrade(tables, tester): op.bulk_insert(tables.externalnotificationevent, [ {'name': 'build_cancelled'}, ]) -def downgrade(tables): +def downgrade(tables, tester): op.execute(tables .externalnotificationevent .delete() diff --git a/data/migrations/versions/a6c463dfb9fe_back_fill_build_expand_config.py b/data/migrations/versions/a6c463dfb9fe_back_fill_build_expand_config.py index 425a20ad2..ff3eaf4b3 100644 --- a/data/migrations/versions/a6c463dfb9fe_back_fill_build_expand_config.py +++ b/data/migrations/versions/a6c463dfb9fe_back_fill_build_expand_config.py @@ -22,7 +22,7 @@ from alembic import op class RepositoryBuildTrigger(BaseModel): config = TextField(default='{}') -def upgrade(tables): +def upgrade(tables, tester): repostioryBuildTriggers = RepositoryBuildTrigger.select() for repositoryBuildTrigger in repostioryBuildTriggers: config = json.loads(repositoryBuildTrigger.config) @@ -30,7 +30,7 @@ def upgrade(tables): repositoryBuildTrigger.save() -def downgrade(tables): +def downgrade(tables, tester): repostioryBuildTriggers = RepositoryBuildTrigger.select() for repositoryBuildTrigger in repostioryBuildTriggers: config = json.loads(repositoryBuildTrigger.config) diff --git a/data/migrations/versions/b4c2d45bc132_add_deleted_namespace_table.py b/data/migrations/versions/b4c2d45bc132_add_deleted_namespace_table.py index e5f137877..271162b95 100644 --- a/data/migrations/versions/b4c2d45bc132_add_deleted_namespace_table.py +++ b/data/migrations/versions/b4c2d45bc132_add_deleted_namespace_table.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.create_table('deletednamespace', sa.Column('id', sa.Integer(), nullable=False), @@ -32,8 +32,19 @@ def upgrade(tables): op.create_index('deletednamespace_queue_id', 'deletednamespace', ['queue_id'], unique=False) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_table('deletednamespace', [ + ('namespace_id', tester.TestDataType.Foreign('user')), + ('marked', tester.TestDataType.DateTime), + ('original_username', tester.TestDataType.UTF8Char), + ('original_email', tester.TestDataType.String), + ('queue_id', tester.TestDataType.Foreign('queueitem')), + ]) + # ### end population of test data ### # -def downgrade(tables): + + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_table('deletednamespace') # ### end Alembic commands ### diff --git a/data/migrations/versions/b4df55dea4b3_add_repository_kind.py b/data/migrations/versions/b4df55dea4b3_add_repository_kind.py index 16123975f..a50e309ef 100644 --- a/data/migrations/versions/b4df55dea4b3_add_repository_kind.py +++ b/data/migrations/versions/b4df55dea4b3_add_repository_kind.py @@ -15,7 +15,7 @@ import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): op.create_table( 'repositorykind', sa.Column('id', sa.Integer(), nullable=False), @@ -36,8 +36,12 @@ def upgrade(tables): op.create_index('repository_kind_id', 'repository', ['kind_id'], unique=False) op.create_foreign_key(op.f('fk_repository_kind_id_repositorykind'), 'repository', 'repositorykind', ['kind_id'], ['id']) + # ### population of test data ### # + tester.populate_column('repository', 'kind_id', tester.TestDataType.Foreign('repositorykind')) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): op.drop_constraint(op.f('fk_repository_kind_id_repositorykind'), 'repository', type_='foreignkey') op.drop_index('repository_kind_id', table_name='repository') op.drop_column(u'repository', 'kind_id') diff --git a/data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py b/data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py index 0530963e7..ae94bdb5c 100644 --- a/data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py +++ b/data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py @@ -15,7 +15,7 @@ import sqlalchemy as sa from util.migrate import UTF8CharField -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.create_table('robotaccountmetadata', sa.Column('id', sa.Integer(), nullable=False), @@ -28,8 +28,16 @@ def upgrade(tables): op.create_index('robotaccountmetadata_robot_account_id', 'robotaccountmetadata', ['robot_account_id'], unique=True) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_table('robotaccountmetadata', [ + ('robot_account_id', tester.TestDataType.Foreign('user')), + ('description', tester.TestDataType.UTF8Char), + ('unstructured_json', tester.TestDataType.JSON), + ]) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_table('robotaccountmetadata') # ### end Alembic commands ### diff --git a/data/migrations/versions/b8ae68ad3e52_change_blobupload_fields_to_bigintegers_.py b/data/migrations/versions/b8ae68ad3e52_change_blobupload_fields_to_bigintegers_.py index 90a5b8978..d36e07c08 100644 --- a/data/migrations/versions/b8ae68ad3e52_change_blobupload_fields_to_bigintegers_.py +++ b/data/migrations/versions/b8ae68ad3e52_change_blobupload_fields_to_bigintegers_.py @@ -14,11 +14,21 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): op.alter_column('blobupload', 'byte_count', existing_type=sa.Integer(), type_=sa.BigInteger()) op.alter_column('blobupload', 'uncompressed_byte_count', existing_type=sa.Integer(), type_=sa.BigInteger()) + # ### population of test data ### # + tester.populate_column('blobupload', 'byte_count', tester.TestDataType.BigInteger) + tester.populate_column('blobupload', 'uncompressed_byte_count', tester.TestDataType.BigInteger) + # ### end population of test data ### # + + +def downgrade(tables, tester): + # ### population of test data ### # + tester.populate_column('blobupload', 'byte_count', tester.TestDataType.Integer) + tester.populate_column('blobupload', 'uncompressed_byte_count', tester.TestDataType.Integer) + # ### end population of test data ### # -def downgrade(tables): op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(), type_=sa.Integer()) op.alter_column('blobupload', 'uncompressed_byte_count', existing_type=sa.BigInteger(), type_=sa.Integer()) diff --git a/data/migrations/versions/be8d1c402ce0_add_teamsync_table.py b/data/migrations/versions/be8d1c402ce0_add_teamsync_table.py index e13d0a344..a8a18cb8d 100644 --- a/data/migrations/versions/be8d1c402ce0_add_teamsync_table.py +++ b/data/migrations/versions/be8d1c402ce0_add_teamsync_table.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from util.migrate import UTF8LongText -def upgrade(tables): +def upgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.create_table('teamsync', sa.Column('id', sa.Integer(), nullable=False), @@ -32,8 +32,18 @@ def upgrade(tables): op.create_index('teamsync_team_id', 'teamsync', ['team_id'], unique=True) ### end Alembic commands ### + # ### population of test data ### # + tester.populate_table('teamsync', [ + ('team_id', tester.TestDataType.Foreign('team')), + ('transaction_id', tester.TestDataType.String), + ('last_updated', tester.TestDataType.DateTime), + ('service_id', tester.TestDataType.Foreign('loginservice')), + ('config', tester.TestDataType.JSON), + ]) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.drop_table('teamsync') ### end Alembic commands ### diff --git a/data/migrations/versions/c156deb8845d_reset_our_migrations_with_a_required_.py b/data/migrations/versions/c156deb8845d_reset_our_migrations_with_a_required_.py index 7e9f1743c..d4ba00591 100644 --- a/data/migrations/versions/c156deb8845d_reset_our_migrations_with_a_required_.py +++ b/data/migrations/versions/c156deb8845d_reset_our_migrations_with_a_required_.py @@ -15,7 +15,7 @@ import sqlalchemy as sa from util.migrate import UTF8LongText, UTF8CharField from datetime import datetime -def upgrade(tables): +def upgrade(tables, tester): now = datetime.now().strftime("'%Y-%m-%d %H:%M:%S'") op.create_table('accesstokenkind', @@ -895,8 +895,305 @@ def upgrade(tables): {'name':'private'}, ]) + # ### population of test data ### # + tester.populate_table('user', [ + ('uuid', tester.TestDataType.UUID), + ('username', tester.TestDataType.String), + ('password_hash', tester.TestDataType.String), + ('email', tester.TestDataType.String), + ('verified', tester.TestDataType.Boolean), + ('organization', tester.TestDataType.Boolean), + ('robot', tester.TestDataType.Boolean), + ('invoice_email', tester.TestDataType.Boolean), + ('invalid_login_attempts', tester.TestDataType.Integer), + ('last_invalid_login', tester.TestDataType.DateTime), + ('removed_tag_expiration_s', tester.TestDataType.Integer), + ('enabled', tester.TestDataType.Boolean), + ('invoice_email_address', tester.TestDataType.String), + ]) -def downgrade(tables): + tester.populate_table('repository', [ + ('namespace_user_id', tester.TestDataType.Foreign('user')), + ('name', tester.TestDataType.String), + ('visibility_id', tester.TestDataType.Foreign('visibility')), + ('description', tester.TestDataType.String), + ('badge_token', tester.TestDataType.String), + ]) + + tester.populate_table('emailconfirmation', [ + ('code', tester.TestDataType.String), + ('user_id', tester.TestDataType.Foreign('user')), + ('pw_reset', tester.TestDataType.Boolean), + ('email_confirm', tester.TestDataType.Boolean), + ('created', tester.TestDataType.DateTime), + ]) + + tester.populate_table('federatedlogin', [ + ('user_id', tester.TestDataType.Foreign('user')), + ('service_id', tester.TestDataType.Foreign('loginservice')), + ('service_ident', tester.TestDataType.String), + ('metadata_json', tester.TestDataType.JSON), + ]) + + tester.populate_table('imagestorage', [ + ('uuid', tester.TestDataType.UUID), + ('checksum', tester.TestDataType.String), + ('image_size', tester.TestDataType.BigInteger), + ('uncompressed_size', tester.TestDataType.BigInteger), + ('uploading', tester.TestDataType.Boolean), + ('cas_path', tester.TestDataType.Boolean), + ('content_checksum', tester.TestDataType.String), + ]) + + tester.populate_table('image', [ + ('docker_image_id', tester.TestDataType.UUID), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('ancestors', tester.TestDataType.String), + ('storage_id', tester.TestDataType.Foreign('imagestorage')), + ('security_indexed', tester.TestDataType.Boolean), + ('security_indexed_engine', tester.TestDataType.Integer), + ]) + + tester.populate_table('imagestorageplacement', [ + ('storage_id', tester.TestDataType.Foreign('imagestorage')), + ('location_id', tester.TestDataType.Foreign('imagestoragelocation')), + ]) + + tester.populate_table('messages', [ + ('content', tester.TestDataType.String), + ('uuid', tester.TestDataType.UUID), + ]) + + tester.populate_table('queueitem', [ + ('queue_name', tester.TestDataType.String), + ('body', tester.TestDataType.JSON), + ('available_after', tester.TestDataType.DateTime), + ('available', tester.TestDataType.Boolean), + ('processing_expires', tester.TestDataType.DateTime), + ('retries_remaining', tester.TestDataType.Integer), + ]) + + tester.populate_table('servicekeyapproval', [ + ('approver_id', tester.TestDataType.Foreign('user')), + ('approval_type', tester.TestDataType.String), + ('approved_date', tester.TestDataType.DateTime), + ('notes', tester.TestDataType.String), + ]) + + tester.populate_table('servicekey', [ + ('name', tester.TestDataType.String), + ('kid', tester.TestDataType.String), + ('service', tester.TestDataType.String), + ('jwk', tester.TestDataType.JSON), + ('metadata', tester.TestDataType.JSON), + ('created_date', tester.TestDataType.DateTime), + ('approval_id', tester.TestDataType.Foreign('servicekeyapproval')), + ]) + + tester.populate_table('label', [ + ('uuid', tester.TestDataType.UUID), + ('key', tester.TestDataType.UTF8Char), + ('value', tester.TestDataType.JSON), + ('media_type_id', tester.TestDataType.Foreign('mediatype')), + ('source_type_id', tester.TestDataType.Foreign('labelsourcetype')), + ]) + + tester.populate_table('logentry', [ + ('kind_id', tester.TestDataType.Foreign('logentrykind')), + ('account_id', tester.TestDataType.Foreign('user')), + ('performer_id', tester.TestDataType.Foreign('user')), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('datetime', tester.TestDataType.DateTime), + ('ip', tester.TestDataType.String), + ('metadata_json', tester.TestDataType.JSON), + ]) + + tester.populate_table('notification', [ + ('uuid', tester.TestDataType.UUID), + ('kind_id', tester.TestDataType.Foreign('notificationkind')), + ('target_id', tester.TestDataType.Foreign('user')), + ('metadata_json', tester.TestDataType.JSON), + ('created', tester.TestDataType.DateTime), + ('dismissed', tester.TestDataType.Boolean), + ('lookup_path', tester.TestDataType.String), + ]) + + tester.populate_table('oauthapplication', [ + ('client_id', tester.TestDataType.String), + ('client_secret', tester.TestDataType.String), + ('redirect_uri', tester.TestDataType.String), + ('application_uri', tester.TestDataType.String), + ('organization_id', tester.TestDataType.Foreign('user')), + ('name', tester.TestDataType.String), + ('description', tester.TestDataType.String), + ]) + + tester.populate_table('team', [ + ('name', tester.TestDataType.String), + ('organization_id', tester.TestDataType.Foreign('user')), + ('role_id', tester.TestDataType.Foreign('teamrole')), + ('description', tester.TestDataType.String), + ]) + + tester.populate_table('torrentinfo', [ + ('storage_id', tester.TestDataType.Foreign('imagestorage')), + ('piece_length', tester.TestDataType.Integer), + ('pieces', tester.TestDataType.String), + ]) + + tester.populate_table('userregion', [ + ('user_id', tester.TestDataType.Foreign('user')), + ('location_id', tester.TestDataType.Foreign('imagestoragelocation')), + ]) + + tester.populate_table('accesstoken', [ + ('friendly_name', tester.TestDataType.String), + ('code', tester.TestDataType.String), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('created', tester.TestDataType.DateTime), + ('role_id', tester.TestDataType.Foreign('role')), + ('temporary', tester.TestDataType.Boolean), + ('kind_id', tester.TestDataType.Foreign('accesstokenkind')), + ]) + + tester.populate_table('blobupload', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('uuid', tester.TestDataType.UUID), + ('byte_count', tester.TestDataType.Integer), + ('sha_state', tester.TestDataType.String), + ('location_id', tester.TestDataType.Foreign('imagestoragelocation')), + ('chunk_count', tester.TestDataType.Integer), + ('created', tester.TestDataType.DateTime), + ]) + + tester.populate_table('oauthaccesstoken', [ + ('uuid', tester.TestDataType.UUID), + ('application_id', tester.TestDataType.Foreign('oauthapplication')), + ('authorized_user_id', tester.TestDataType.Foreign('user')), + ('scope', tester.TestDataType.String), + ('access_token', tester.TestDataType.String), + ('token_type', tester.TestDataType.String), + ('expires_at', tester.TestDataType.DateTime), + ('data', tester.TestDataType.JSON), + ]) + + tester.populate_table('oauthauthorizationcode', [ + ('application_id', tester.TestDataType.Foreign('oauthapplication')), + ('code', tester.TestDataType.String), + ('scope', tester.TestDataType.String), + ('data', tester.TestDataType.JSON), + ]) + + tester.populate_table('permissionprototype', [ + ('org_id', tester.TestDataType.Foreign('user')), + ('uuid', tester.TestDataType.UUID), + ('activating_user_id', tester.TestDataType.Foreign('user')), + ('delegate_user_id', tester.TestDataType.Foreign('user')), + ('role_id', tester.TestDataType.Foreign('role')), + ]) + + tester.populate_table('repositoryactioncount', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('count', tester.TestDataType.Integer), + ('date', tester.TestDataType.Date), + ]) + + tester.populate_table('repositoryauthorizedemail', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('email', tester.TestDataType.String), + ('code', tester.TestDataType.String), + ('confirmed', tester.TestDataType.Boolean), + ]) + + tester.populate_table('repositorynotification', [ + ('uuid', tester.TestDataType.UUID), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('event_id', tester.TestDataType.Foreign('externalnotificationevent')), + ('method_id', tester.TestDataType.Foreign('externalnotificationmethod')), + ('title', tester.TestDataType.String), + ('config_json', tester.TestDataType.JSON), + ('event_config_json', tester.TestDataType.JSON), + ]) + + tester.populate_table('repositorypermission', [ + ('team_id', tester.TestDataType.Foreign('team')), + ('user_id', tester.TestDataType.Foreign('user')), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('role_id', tester.TestDataType.Foreign('role')), + ]) + + tester.populate_table('star', [ + ('user_id', tester.TestDataType.Foreign('user')), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('created', tester.TestDataType.DateTime), + ]) + + tester.populate_table('teammember', [ + ('user_id', tester.TestDataType.Foreign('user')), + ('team_id', tester.TestDataType.Foreign('team')), + ]) + + tester.populate_table('teammemberinvite', [ + ('user_id', tester.TestDataType.Foreign('user')), + ('email', tester.TestDataType.String), + ('team_id', tester.TestDataType.Foreign('team')), + ('inviter_id', tester.TestDataType.Foreign('user')), + ('invite_token', tester.TestDataType.String), + ]) + + tester.populate_table('derivedstorageforimage', [ + ('source_image_id', tester.TestDataType.Foreign('image')), + ('derivative_id', tester.TestDataType.Foreign('imagestorage')), + ('transformation_id', tester.TestDataType.Foreign('imagestoragetransformation')), + ('uniqueness_hash', tester.TestDataType.String), + ]) + + tester.populate_table('repositorybuildtrigger', [ + ('uuid', tester.TestDataType.UUID), + ('service_id', tester.TestDataType.Foreign('buildtriggerservice')), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('connected_user_id', tester.TestDataType.Foreign('user')), + ('auth_token', tester.TestDataType.String), + ('config', tester.TestDataType.JSON), + ]) + + tester.populate_table('repositorytag', [ + ('name', tester.TestDataType.String), + ('image_id', tester.TestDataType.Foreign('image')), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('lifetime_start_ts', tester.TestDataType.Integer), + ('hidden', tester.TestDataType.Boolean), + ('reversion', tester.TestDataType.Boolean), + ]) + + tester.populate_table('repositorybuild', [ + ('uuid', tester.TestDataType.UUID), + ('phase', tester.TestDataType.String), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('access_token_id', tester.TestDataType.Foreign('accesstoken')), + ('resource_key', tester.TestDataType.String), + ('job_config', tester.TestDataType.JSON), + ('started', tester.TestDataType.DateTime), + ('display_name', tester.TestDataType.JSON), + ('trigger_id', tester.TestDataType.Foreign('repositorybuildtrigger')), + ('logs_archived', tester.TestDataType.Boolean), + ]) + + tester.populate_table('tagmanifest', [ + ('tag_id', tester.TestDataType.Foreign('repositorytag')), + ('digest', tester.TestDataType.String), + ('json_data', tester.TestDataType.JSON), + ]) + + tester.populate_table('tagmanifestlabel', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('annotated_id', tester.TestDataType.Foreign('tagmanifest')), + ('label_id', tester.TestDataType.Foreign('label')), + ]) + # ### end population of test data ### # + + +def downgrade(tables, tester): op.drop_table('tagmanifestlabel') op.drop_table('tagmanifest') op.drop_table('repositorybuild') diff --git a/data/migrations/versions/c3d4b7ebcdf7_backfill_repositorysearchscore_table.py b/data/migrations/versions/c3d4b7ebcdf7_backfill_repositorysearchscore_table.py index 5722c0926..64c285aa0 100644 --- a/data/migrations/versions/c3d4b7ebcdf7_backfill_repositorysearchscore_table.py +++ b/data/migrations/versions/c3d4b7ebcdf7_backfill_repositorysearchscore_table.py @@ -13,11 +13,11 @@ down_revision = 'f30984525c86' from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): # Add a 0 entry into the RepositorySearchScore table for each repository that isn't present conn = op.get_bind() conn.execute("insert into repositorysearchscore (repository_id, score) SELECT id, 0 FROM " + "repository WHERE id not in (select repository_id from repositorysearchscore)") -def downgrade(tables): +def downgrade(tables, tester): pass diff --git a/data/migrations/versions/c91c564aad34_drop_checksum_on_imagestorage.py b/data/migrations/versions/c91c564aad34_drop_checksum_on_imagestorage.py index d75cc7b32..f4cfe7bc3 100644 --- a/data/migrations/versions/c91c564aad34_drop_checksum_on_imagestorage.py +++ b/data/migrations/versions/c91c564aad34_drop_checksum_on_imagestorage.py @@ -14,9 +14,9 @@ from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): op.drop_column('imagestorage', 'checksum') -def downgrade(tables): +def downgrade(tables, tester): op.add_column('imagestorage', sa.Column('checksum', sa.String(length=255), nullable=True)) diff --git a/data/migrations/versions/cbc8177760d9_add_user_location_field.py b/data/migrations/versions/cbc8177760d9_add_user_location_field.py index b88f6da42..23f24a9dc 100644 --- a/data/migrations/versions/cbc8177760d9_add_user_location_field.py +++ b/data/migrations/versions/cbc8177760d9_add_user_location_field.py @@ -15,9 +15,13 @@ import sqlalchemy as sa from sqlalchemy.dialects import mysql from util.migrate import UTF8CharField -def upgrade(tables): +def upgrade(tables, tester): op.add_column('user', sa.Column('location', UTF8CharField(length=255), nullable=True)) + # ### population of test data ### # + tester.populate_column('user', 'location', tester.TestDataType.UTF8Char) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): op.drop_column('user', 'location') diff --git a/data/migrations/versions/d42c175b439a_backfill_state_id_and_make_it_unique.py b/data/migrations/versions/d42c175b439a_backfill_state_id_and_make_it_unique.py index 2f65c57fd..979702124 100644 --- a/data/migrations/versions/d42c175b439a_backfill_state_id_and_make_it_unique.py +++ b/data/migrations/versions/d42c175b439a_backfill_state_id_and_make_it_unique.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): # Backfill the queueitem table's state_id field with unique values for all entries which are # empty. conn = op.get_bind() @@ -26,7 +26,7 @@ def upgrade(tables): # ### end Alembic commands ### -def downgrade(tables): +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_index('queueitem_state_id', table_name='queueitem') op.create_index('queueitem_state_id', 'queueitem', ['state_id'], unique=False) diff --git a/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py b/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py index 9ae981008..a5ad2de6e 100644 --- a/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py +++ b/data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py @@ -12,13 +12,13 @@ down_revision = 'dc4af11a5f90' from alembic import op -def upgrade(tables): +def upgrade(tables, tester): op.bulk_insert(tables.logentrykind, [ {'name': 'change_tag_expiration'}, ]) -def downgrade(tables): +def downgrade(tables, tester): op.execute(tables .logentrykind .delete() diff --git a/data/migrations/versions/dc4af11a5f90_add_notification_number_of_failures_.py b/data/migrations/versions/dc4af11a5f90_add_notification_number_of_failures_.py index ffde9d687..93bdf4c56 100644 --- a/data/migrations/versions/dc4af11a5f90_add_notification_number_of_failures_.py +++ b/data/migrations/versions/dc4af11a5f90_add_notification_number_of_failures_.py @@ -14,7 +14,7 @@ import sqlalchemy as sa from alembic import op -def upgrade(tables): +def upgrade(tables, tester): op.add_column('repositorynotification', sa.Column('number_of_failures', sa.Integer(), nullable=False, @@ -23,8 +23,12 @@ def upgrade(tables): {'name': 'reset_repo_notification'}, ]) + # ### population of test data ### # + tester.populate_column('repositorynotification', 'number_of_failures', tester.TestDataType.Integer) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): op.drop_column('repositorynotification', 'number_of_failures') op.execute(tables .logentrykind diff --git a/data/migrations/versions/e2894a3a3c19_add_full_text_search_indexing_for_repo_.py b/data/migrations/versions/e2894a3a3c19_add_full_text_search_indexing_for_repo_.py index 9395fd926..281e148c8 100644 --- a/data/migrations/versions/e2894a3a3c19_add_full_text_search_indexing_for_repo_.py +++ b/data/migrations/versions/e2894a3a3c19_add_full_text_search_indexing_for_repo_.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): if op.get_bind().engine.name == 'postgresql': op.execute('CREATE EXTENSION IF NOT EXISTS pg_trgm') @@ -24,7 +24,7 @@ def upgrade(tables): # ### end Alembic commands ### -def downgrade(tables): +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_index('repository_name__fulltext', table_name='repository') op.drop_index('repository_description__fulltext', table_name='repository') diff --git a/data/migrations/versions/ed01e313d3cb_add_trust_enabled_to_repository.py b/data/migrations/versions/ed01e313d3cb_add_trust_enabled_to_repository.py index 64ca81c56..32a4ee66b 100644 --- a/data/migrations/versions/ed01e313d3cb_add_trust_enabled_to_repository.py +++ b/data/migrations/versions/ed01e313d3cb_add_trust_enabled_to_repository.py @@ -13,7 +13,7 @@ down_revision = 'c3d4b7ebcdf7' from alembic import op import sqlalchemy as sa -def upgrade(tables): +def upgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.add_column('repository', sa.Column('trust_enabled', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) ### end Alembic commands ### @@ -21,8 +21,12 @@ def upgrade(tables): {'name': 'change_repo_trust'}, ]) + # ### population of test data ### # + tester.populate_column('repository', 'trust_enabled', tester.TestDataType.Boolean) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.drop_column('repository', 'trust_enabled') ### end Alembic commands ### diff --git a/data/migrations/versions/f30984525c86_add_repositorysearchscore_table.py b/data/migrations/versions/f30984525c86_add_repositorysearchscore_table.py index 48795acff..c2291ebc5 100644 --- a/data/migrations/versions/f30984525c86_add_repositorysearchscore_table.py +++ b/data/migrations/versions/f30984525c86_add_repositorysearchscore_table.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.create_table('repositorysearchscore', sa.Column('id', sa.Integer(), nullable=False), @@ -28,8 +28,16 @@ def upgrade(tables): op.create_index('repositorysearchscore_score', 'repositorysearchscore', ['score'], unique=False) ### end Alembic commands ### + # ### population of test data ### # + tester.populate_table('repositorysearchscore', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('score', tester.TestDataType.BigInteger), + ('last_updated', tester.TestDataType.DateTime), + ]) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.drop_table('repositorysearchscore') ### end Alembic commands ### diff --git a/data/migrations/versions/f5167870dd66_update_queue_item_table_indices.py b/data/migrations/versions/f5167870dd66_update_queue_item_table_indices.py index 2131de04b..7204fe159 100644 --- a/data/migrations/versions/f5167870dd66_update_queue_item_table_indices.py +++ b/data/migrations/versions/f5167870dd66_update_queue_item_table_indices.py @@ -14,7 +14,7 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.create_index('queueitem_processing_expires_available', 'queueitem', ['processing_expires', 'available'], unique=False) op.create_index('queueitem_pe_aafter_qname_rremaining_available', 'queueitem', ['processing_expires', 'available_after', 'queue_name', 'retries_remaining', 'available'], unique=False) @@ -27,7 +27,7 @@ def upgrade(tables): ### end Alembic commands ### -def downgrade(tables): +def downgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.create_index('queueitem_retries_remaining', 'queueitem', ['retries_remaining'], unique=False) op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False) diff --git a/data/migrations/versions/faf752bd2e0a_add_user_metadata_fields.py b/data/migrations/versions/faf752bd2e0a_add_user_metadata_fields.py index 8822ff06a..ea52978d1 100644 --- a/data/migrations/versions/faf752bd2e0a_add_user_metadata_fields.py +++ b/data/migrations/versions/faf752bd2e0a_add_user_metadata_fields.py @@ -15,7 +15,7 @@ import sqlalchemy as sa from util.migrate import UTF8CharField -def upgrade(tables): +def upgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.add_column('user', sa.Column('company', UTF8CharField(length=255), nullable=True)) op.add_column('user', sa.Column('family_name', UTF8CharField(length=255), nullable=True)) @@ -28,8 +28,14 @@ def upgrade(tables): {'name':'enter_company'}, ]) + # ### population of test data ### # + tester.populate_column('user', 'company', tester.TestDataType.UTF8Char) + tester.populate_column('user', 'family_name', tester.TestDataType.UTF8Char) + tester.populate_column('user', 'given_name', tester.TestDataType.UTF8Char) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): ### commands auto generated by Alembic - please adjust! ### op.drop_column('user', 'given_name') op.drop_column('user', 'family_name') diff --git a/data/migrations/versions/fc47c1ec019f_add_state_id_field_to_queueitem.py b/data/migrations/versions/fc47c1ec019f_add_state_id_field_to_queueitem.py index 7b9085521..fbe5a31f1 100644 --- a/data/migrations/versions/fc47c1ec019f_add_state_id_field_to_queueitem.py +++ b/data/migrations/versions/fc47c1ec019f_add_state_id_field_to_queueitem.py @@ -14,14 +14,18 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.add_column('queueitem', sa.Column('state_id', sa.String(length=255), nullable=False, server_default='')) op.create_index('queueitem_state_id', 'queueitem', ['state_id'], unique=False) # ### end Alembic commands ### + # ### population of test data ### # + tester.populate_column('queueitem', 'state_id', tester.TestDataType.String) + # ### end population of test data ### # -def downgrade(tables): + +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_index('queueitem_state_id', table_name='queueitem') op.drop_column('queueitem', 'state_id') diff --git a/util/migrate/cleanup_old_robots.py b/util/migrate/cleanup_old_robots.py index 00b6c8289..e35b3540a 100644 --- a/util/migrate/cleanup_old_robots.py +++ b/util/migrate/cleanup_old_robots.py @@ -19,8 +19,11 @@ def cleanup_old_robots(page_size=50): for robot in list(User.select().where(User.robot == True).paginate(page_number, page_size)): found_bots = True logger.info("Checking robot %s (page %s)", robot.username, page_number) - namespace, _ = parse_robot_username(robot.username) + parsed = parse_robot_username(robot.username) + if parsed is None: + continue + namespace, _ = parsed if namespace in encountered_namespaces: if not encountered_namespaces[namespace]: logger.info('Marking %s to be deleted', robot.username)