Merge pull request #3069 from quay/joseph.schorr/QUAY-913/db-test-data

Add support for populating test data during migration testing

data/migrations/env.py

@@ -12,6 +12,8 @@ from urllib import unquote, quote
 from peewee import SqliteDatabase
 
 from data.database import all_models, db
+from data.migrations.tester import NoopTester, PopulateTestDataTester
+
 from app import app
 from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
 from release import GIT_HEAD, REGION, SERVICE
@@ -39,6 +41,18 @@ tables = AttrDict(target_metadata.tables)
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
+def get_tester():
+    """ Returns the tester to use. We only return the tester that populates data
+        if the TEST_MIGRATE env var is set to `true` AND we make sure we're not
+        connecting to a production database.
+    """
+    if os.environ.get('TEST_MIGRATE', '') == 'true':
+        url = unquote(app.config['DB_URI'])
+        if url.find('.quay.io') < 0:
+            return PopulateTestDataTester()
+
+    return NoopTester()
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -55,7 +69,8 @@ def run_migrations_offline():
     context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True)
 
     with context.begin_transaction():
-        context.run_migrations(tables=tables)
+        context.run_migrations(tables=tables, tester=get_tester())
+
 
 def run_migrations_online():
     """Run migrations in 'online' mode.
@@ -84,7 +99,7 @@ def run_migrations_online():
     try:
         with context.begin_transaction():
             try:
-                context.run_migrations(tables=tables)
+                context.run_migrations(tables=tables, tester=get_tester())
             except (CommandError, ResolutionError) as ex:
                 if 'No such revision' not in str(ex):
                     raise

data/migrations/migration.sh

@@ -8,7 +8,7 @@ PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\
 
 up_mysql() {
   # Run a SQL database on port 3306 inside of Docker.
-  docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql
+  docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql:5.7
 
   echo 'Sleeping for 25...'
   sleep 25
@@ -18,8 +18,8 @@ up_mysql() {
 }
 
 down_mysql() {
-  docker kill mysql
-  docker rm -v mysql
+  docker kill mysql || true
+  docker rm -v mysql || true
 }
 
 up_mariadb() {
@@ -34,8 +34,8 @@ up_mariadb() {
 }
 
 down_mariadb() {
-  docker kill mariadb
-  docker rm -v mariadb
+  docker kill mariadb || true
+  docker rm -v mariadb || true
 }
 
 up_percona() {
@@ -50,8 +50,8 @@ up_percona() {
 }
 
 down_percona() {
-  docker kill percona
-  docker rm -v percona
+  docker kill percona || true
+  docker rm -v percona || true
 }
 
 up_postgres() {
@@ -67,8 +67,8 @@ up_postgres() {
 }
 
 down_postgres() {
-  docker kill postgres
-  docker rm -v postgres
+  docker kill postgres || true
+  docker rm -v postgres || true
 }
 
 gen_migrate() {
@@ -83,14 +83,19 @@ gen_migrate() {
 test_migrate() {
   # Generate a database with the schema as defined by the existing alembic model.
   echo '> Running upgrade'
-  QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head
+  TEST_MIGRATE=true QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head
 
   # Downgrade to verify it works in both directions.
   echo '> Running downgrade'
   COUNT=`ls data/migrations/versions/*.py | wc -l | tr -d ' '`
-  QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT"
+  TEST_MIGRATE=true QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT"
 }
 
+down_mysql
+down_postgres
+down_mariadb
+down_percona
+
 # Test (and generate, if requested) via MySQL.
 echo '> Starting MySQL'
 up_mysql

data/migrations/script.py.mako

@@ -14,9 +14,9 @@ from alembic import op
 import sqlalchemy as sa
 ${imports if imports else ""}
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ${upgrades if upgrades else "pass"}
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     ${downgrades if downgrades else "pass"}

data/migrations/tester.py

@@ -0,0 +1,122 @@
+import json
+import logging
+import uuid
+
+from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from six import add_metaclass
+
+from alembic import op
+from sqlalchemy import text
+
+from util.abchelpers import nooper
+
+logger = logging.getLogger(__name__)
+
+def escape_table_name(table_name):
+  if op.get_bind().engine.name == 'postgresql':
+    # Needed for the `user` table.
+    return '"%s"' % table_name
+
+  return table_name
+
+
+class DataTypes(object):
+  @staticmethod
+  def DateTime():
+    return datetime.now()
+
+  @staticmethod
+  def Date():
+    return datetime.now()
+
+  @staticmethod
+  def String():
+    return 'somestringvalue'
+
+  @staticmethod
+  def UTF8Char():
+    return 'some other value'
+
+  @staticmethod
+  def UUID():
+    return str(uuid.uuid4())
+
+  @staticmethod
+  def JSON():
+    return json.dumps(dict(foo='bar', baz='meh'))
+
+  @staticmethod
+  def Boolean():
+    if op.get_bind().engine.name == 'postgresql':
+      return True
+
+    return 1
+
+  @staticmethod
+  def BigInteger():
+    return 21474836470
+
+  @staticmethod
+  def Integer():
+    return 42
+
+  @staticmethod
+  def Foreign(table_name):
+    def get_index():
+      result = op.get_bind().execute("SELECT id FROM %s LIMIT 1" % escape_table_name(table_name))
+      try:
+        return list(result)[0][0]
+      except IndexError:
+        raise Exception('Could not find row for table %s' % table_name)
+      finally:
+        result.close()
+
+    return get_index
+
+
+@add_metaclass(ABCMeta)
+class MigrationTester(object):
+  """ Implements an interface for adding testing capabilities to the
+      data model migration system in Alembic.
+  """
+  TestDataType = DataTypes
+
+  @abstractmethod
+  def populate_table(self, table_name, fields):
+    """ Called to populate a table with the given fields filled in with testing data. """
+
+  @abstractmethod
+  def populate_column(self, table_name, col_name, field_type):
+    """ Called to populate a column in a table to be filled in with testing data. """
+
+
+@nooper
+class NoopTester(MigrationTester):
+  """ No-op version of the tester, designed for production workloads. """
+
+
+class PopulateTestDataTester(MigrationTester):
+  def populate_table(self, table_name, fields):
+    columns = {field_name: field_type() for field_name, field_type in fields}
+    field_name_vars = [':' + field_name for field_name, _ in fields]
+
+    if op.get_bind().engine.name == 'postgresql':
+      field_names = ["%s" % field_name for field_name, _ in fields]
+    else:
+      field_names = ["`%s`" % field_name for field_name, _ in fields]
+
+    table_name = escape_table_name(table_name)
+    query = text('INSERT INTO %s (%s) VALUES (%s)' % (table_name, ', '.join(field_names),
+                                                      ', '.join(field_name_vars)))
+    logger.info("Executing test query %s with values %s", query, columns.values())
+    op.get_bind().execute(query, **columns)
+
+  def populate_column(self, table_name, col_name, field_type):
+    col_value = field_type()
+    row_id = DataTypes.Foreign(table_name)()
+
+    table_name = escape_table_name(table_name)
+    update_text = text("UPDATE %s SET %s=:col_value where ID=:row_id" % (table_name, col_name))
+    logger.info("Executing test query %s with value %s on row %s", update_text, col_value, row_id)
+    op.get_bind().execute(update_text, col_value=col_value, row_id=row_id)

data/migrations/versions/0cf50323c78b_add_creation_date_to_user_table.py

@@ -14,13 +14,17 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('creation_date', sa.DateTime(), nullable=True))
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'creation_date', tester.TestDataType.DateTime)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('user', 'creation_date')
     # ### end Alembic commands ###

data/migrations/versions/152bb29a1bb3_add_maximum_build_queue_count_setting_.py

@@ -14,13 +14,17 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('maximum_queued_builds_count', sa.Integer(), nullable=True))
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'maximum_queued_builds_count', tester.TestDataType.Integer)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('user', 'maximum_queued_builds_count')
     # ### end Alembic commands ###

data/migrations/versions/152edccba18c_make_blodupload_byte_count_not_nullable.py

@@ -14,11 +14,11 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(),
                     nullable=False)
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(),
                     nullable=True)

data/migrations/versions/1783530bee68_add_logentry2_table_quay_io_only.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('logentry2',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -40,7 +40,7 @@ def upgrade(tables):
     # ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('logentry2')
     # ### end Alembic commands ###

data/migrations/versions/17aff2e1354e_add_automatic_disable_of_build_triggers.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('repositorybuildtrigger', sa.Column('successive_failure_count', sa.Integer(), server_default='0', nullable=False))
     op.add_column('repositorybuildtrigger', sa.Column('successive_internal_error_count', sa.Integer(), server_default='0', nullable=False))
@@ -28,7 +28,13 @@ def upgrade(tables):
         ],
     )
 
-def downgrade(tables):
+    # ### population of test data ### #
+    tester.populate_column('repositorybuildtrigger', 'successive_failure_count', tester.TestDataType.Integer)
+    tester.populate_column('repositorybuildtrigger', 'successive_internal_error_count', tester.TestDataType.Integer)
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('repositorybuildtrigger', 'successive_internal_error_count')
     op.drop_column('repositorybuildtrigger', 'successive_failure_count')

data/migrations/versions/224ce4c72c2f_add_last_accessed_field_to_user_table.py

@@ -14,14 +14,18 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('last_accessed', sa.DateTime(), nullable=True))
     op.create_index('user_last_accessed', 'user', ['last_accessed'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'last_accessed', tester.TestDataType.DateTime)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('user_last_accessed', table_name='user')
     op.drop_column('user', 'last_accessed')

data/migrations/versions/3e8cc74a1e7b_add_severity_and_media_type_to_global_.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('messages', sa.Column('media_type_id', sa.Integer(), nullable=False, server_default='1'))
     op.add_column('messages', sa.Column('severity', sa.String(length=255), nullable=False, server_default='info'))
@@ -33,8 +33,14 @@ def upgrade(tables):
                      {'name': 'text/markdown'},
                    ])
 
+    # ### population of test data ### #
+    tester.populate_column('messages', 'media_type_id', tester.TestDataType.Foreign('mediatype'))
+    tester.populate_column('messages', 'severity', lambda: 'info')
+    tester.populate_column('messages', 'uuid', tester.TestDataType.UUID)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_constraint(op.f('fk_messages_media_type_id_mediatype'), 'messages', type_='foreignkey')
     op.drop_index('messages_uuid', table_name='messages')

data/migrations/versions/45fd8b9869d4_add_notification_type.py

@@ -12,14 +12,14 @@ down_revision = '94836b099894'
 
 from alembic import op
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.bulk_insert(tables.notificationkind,
                  [
                    {'name': 'build_cancelled'},
                  ])
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   op.execute(tables
              .notificationkind
              .delete()

data/migrations/versions/53e2ac668296_remove_reference_to_subdir.py

@@ -33,11 +33,11 @@ def run_migration(migrate_function):
       log.warning("Failed to update build trigger %s with exception: ", trigger[0], e)
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   run_migration(delete_subdir)
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   run_migration(add_subdir)
 
 

data/migrations/versions/5b7503aada1b_cleanup_old_robots.py

@@ -15,9 +15,9 @@ import sqlalchemy as sa
 
 from util.migrate.cleanup_old_robots import cleanup_old_robots
 
-def upgrade(tables):
+def upgrade(tables, tester):
   cleanup_old_robots()
 
-def downgrade(tables):
+def downgrade(tables, tester):
   # Nothing to do.
   pass

data/migrations/versions/61cadbacb9fc_add_ability_for_build_triggers_to_be_.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('disablereason',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -40,8 +40,13 @@ def upgrade(tables):
     op.create_foreign_key(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', 'disablereason', ['disabled_reason_id'], ['id'])
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('repositorybuildtrigger', 'disabled_reason_id', tester.TestDataType.Foreign('disablereason'))
+    tester.populate_column('repositorybuildtrigger', 'enabled', tester.TestDataType.Boolean)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_constraint(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', type_='foreignkey')
     op.drop_index('repositorybuildtrigger_disabled_reason_id', table_name='repositorybuildtrigger')

data/migrations/versions/6c7014e84a5e_add_user_prompt_support.py

@@ -13,7 +13,7 @@ down_revision = 'c156deb8845d'
 from alembic import op
 import sqlalchemy as sa
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table('userpromptkind',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -39,8 +39,14 @@ def upgrade(tables):
         {'name':'confirm_username'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_table('userprompt', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('kind_id', tester.TestDataType.Foreign('userpromptkind')),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('userprompt')
     op.drop_table('userpromptkind')

data/migrations/versions/7367229b38d9_add_support_for_app_specific_tokens.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 from util.migrate import UTF8CharField
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('appspecificauthtoken',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -40,7 +40,20 @@ def upgrade(tables):
         {'name': 'revoke_app_specific_token'},
     ])
 
-def downgrade(tables):
+    # ### population of test data ### #
+    tester.populate_table('appspecificauthtoken', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('uuid', tester.TestDataType.UUID),
+        ('title', tester.TestDataType.UTF8Char),
+        ('token_code', tester.TestDataType.String),
+        ('created', tester.TestDataType.DateTime),
+        ('expiration', tester.TestDataType.DateTime),
+        ('last_accessed', tester.TestDataType.DateTime),
+    ])
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('appspecificauthtoken')
     # ### end Alembic commands ###

data/migrations/versions/7a525c68eb13_add_oci_app_models.py

@@ -16,7 +16,7 @@ from sqlalchemy.dialects import mysql
 from util.migrate import UTF8LongText, UTF8CharField
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.create_table(
     'tagkind',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -307,7 +307,7 @@ def upgrade(tables):
   )
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   op.drop_table('manifestlayerscan')
   op.drop_table('manifestlayerdockerv1')
   op.drop_table('tag')

data/migrations/versions/87fbbc224f10_add_disabled_datetime_to_trigger.py

@@ -14,14 +14,18 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('repositorybuildtrigger', sa.Column('disabled_datetime', sa.DateTime(), nullable=True))
     op.create_index('repositorybuildtrigger_disabled_datetime', 'repositorybuildtrigger', ['disabled_datetime'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('repositorybuildtrigger', 'disabled_datetime', tester.TestDataType.DateTime)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('repositorybuildtrigger_disabled_datetime', table_name='repositorybuildtrigger')
     op.drop_column('repositorybuildtrigger', 'disabled_datetime')

data/migrations/versions/94836b099894_create_new_notification_type.py

@@ -13,14 +13,14 @@ down_revision = 'faf752bd2e0a'
 from alembic import op
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.bulk_insert(tables.externalnotificationevent,
                    [
                        {'name': 'build_cancelled'},
                    ])
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     op.execute(tables
                .externalnotificationevent
                .delete()

data/migrations/versions/a6c463dfb9fe_back_fill_build_expand_config.py

@@ -22,7 +22,7 @@ from alembic import op
 class RepositoryBuildTrigger(BaseModel):
   config = TextField(default='{}')
 
-def upgrade(tables):
+def upgrade(tables, tester):
   repostioryBuildTriggers = RepositoryBuildTrigger.select()
   for repositoryBuildTrigger in repostioryBuildTriggers:
     config = json.loads(repositoryBuildTrigger.config)
@@ -30,7 +30,7 @@ def upgrade(tables):
     repositoryBuildTrigger.save()
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   repostioryBuildTriggers = RepositoryBuildTrigger.select()
   for repositoryBuildTrigger in repostioryBuildTriggers:
     config = json.loads(repositoryBuildTrigger.config)

data/migrations/versions/b4c2d45bc132_add_deleted_namespace_table.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('deletednamespace',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -32,8 +32,19 @@ def upgrade(tables):
     op.create_index('deletednamespace_queue_id', 'deletednamespace', ['queue_id'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('deletednamespace', [
+        ('namespace_id', tester.TestDataType.Foreign('user')),
+        ('marked', tester.TestDataType.DateTime),
+        ('original_username', tester.TestDataType.UTF8Char),
+        ('original_email', tester.TestDataType.String),
+        ('queue_id', tester.TestDataType.Foreign('queueitem')),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('deletednamespace')
     # ### end Alembic commands ###

data/migrations/versions/b4df55dea4b3_add_repository_kind.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.create_table(
     'repositorykind',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -36,8 +36,12 @@ def upgrade(tables):
   op.create_index('repository_kind_id', 'repository', ['kind_id'], unique=False)
   op.create_foreign_key(op.f('fk_repository_kind_id_repositorykind'), 'repository', 'repositorykind', ['kind_id'], ['id'])
 
+  # ### population of test data ### #
+  tester.populate_column('repository', 'kind_id', tester.TestDataType.Foreign('repositorykind'))
+  # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
   op.drop_constraint(op.f('fk_repository_kind_id_repositorykind'), 'repository', type_='foreignkey')
   op.drop_index('repository_kind_id', table_name='repository')
   op.drop_column(u'repository', 'kind_id')

data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from util.migrate import UTF8CharField
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('robotaccountmetadata',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -28,8 +28,16 @@ def upgrade(tables):
     op.create_index('robotaccountmetadata_robot_account_id', 'robotaccountmetadata', ['robot_account_id'], unique=True)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('robotaccountmetadata', [
+        ('robot_account_id', tester.TestDataType.Foreign('user')),        
+        ('description', tester.TestDataType.UTF8Char),        
+        ('unstructured_json', tester.TestDataType.JSON),        
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('robotaccountmetadata')
     # ### end Alembic commands ###

data/migrations/versions/b8ae68ad3e52_change_blobupload_fields_to_bigintegers_.py

@@ -14,11 +14,21 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.Integer(), type_=sa.BigInteger())
     op.alter_column('blobupload', 'uncompressed_byte_count', existing_type=sa.Integer(), type_=sa.BigInteger())
 
+    # ### population of test data ### #
+    tester.populate_column('blobupload', 'byte_count', tester.TestDataType.BigInteger)
+    tester.populate_column('blobupload', 'uncompressed_byte_count', tester.TestDataType.BigInteger)
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
+    # ### population of test data ### #
+    tester.populate_column('blobupload', 'byte_count', tester.TestDataType.Integer)
+    tester.populate_column('blobupload', 'uncompressed_byte_count', tester.TestDataType.Integer)
+    # ### end population of test data ### #
 
-def downgrade(tables):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(), type_=sa.Integer())
     op.alter_column('blobupload', 'uncompressed_byte_count', existing_type=sa.BigInteger(), type_=sa.Integer())

data/migrations/versions/be8d1c402ce0_add_teamsync_table.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from util.migrate import UTF8LongText
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table('teamsync',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -32,8 +32,18 @@ def upgrade(tables):
     op.create_index('teamsync_team_id', 'teamsync', ['team_id'], unique=True)
     ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('teamsync', [
+        ('team_id', tester.TestDataType.Foreign('team')),
+        ('transaction_id', tester.TestDataType.String),
+        ('last_updated', tester.TestDataType.DateTime),
+        ('service_id', tester.TestDataType.Foreign('loginservice')),
+        ('config', tester.TestDataType.JSON),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('teamsync')
     ### end Alembic commands ###

data/migrations/versions/c156deb8845d_reset_our_migrations_with_a_required_.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from util.migrate import UTF8LongText, UTF8CharField
 from datetime import datetime
 
-def upgrade(tables):
+def upgrade(tables, tester):
     now = datetime.now().strftime("'%Y-%m-%d %H:%M:%S'")
 
     op.create_table('accesstokenkind',
@@ -895,8 +895,305 @@ def upgrade(tables):
         {'name':'private'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_table('user', [
+        ('uuid', tester.TestDataType.UUID),
+        ('username', tester.TestDataType.String),
+        ('password_hash', tester.TestDataType.String),
+        ('email', tester.TestDataType.String),
+        ('verified', tester.TestDataType.Boolean),
+        ('organization', tester.TestDataType.Boolean),
+        ('robot', tester.TestDataType.Boolean),
+        ('invoice_email', tester.TestDataType.Boolean),
+        ('invalid_login_attempts', tester.TestDataType.Integer),
+        ('last_invalid_login', tester.TestDataType.DateTime),
+        ('removed_tag_expiration_s', tester.TestDataType.Integer),
+        ('enabled', tester.TestDataType.Boolean),
+        ('invoice_email_address', tester.TestDataType.String),
+    ])
 
-def downgrade(tables):
+    tester.populate_table('repository', [
+        ('namespace_user_id', tester.TestDataType.Foreign('user')),
+        ('name', tester.TestDataType.String),
+        ('visibility_id', tester.TestDataType.Foreign('visibility')),
+        ('description', tester.TestDataType.String),
+        ('badge_token', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('emailconfirmation', [
+        ('code', tester.TestDataType.String),
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('pw_reset', tester.TestDataType.Boolean),
+        ('email_confirm', tester.TestDataType.Boolean),
+        ('created', tester.TestDataType.DateTime),
+    ])
+
+    tester.populate_table('federatedlogin', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('service_id', tester.TestDataType.Foreign('loginservice')),
+        ('service_ident', tester.TestDataType.String),
+        ('metadata_json', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('imagestorage', [
+        ('uuid', tester.TestDataType.UUID),
+        ('checksum', tester.TestDataType.String),
+        ('image_size', tester.TestDataType.BigInteger),
+        ('uncompressed_size', tester.TestDataType.BigInteger),
+        ('uploading', tester.TestDataType.Boolean),
+        ('cas_path', tester.TestDataType.Boolean),
+        ('content_checksum', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('image', [
+        ('docker_image_id', tester.TestDataType.UUID),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('ancestors', tester.TestDataType.String),
+        ('storage_id', tester.TestDataType.Foreign('imagestorage')),
+        ('security_indexed', tester.TestDataType.Boolean),
+        ('security_indexed_engine', tester.TestDataType.Integer),
+    ])
+
+    tester.populate_table('imagestorageplacement', [
+        ('storage_id', tester.TestDataType.Foreign('imagestorage')),
+        ('location_id', tester.TestDataType.Foreign('imagestoragelocation')),
+    ])
+
+    tester.populate_table('messages', [
+        ('content', tester.TestDataType.String),
+        ('uuid', tester.TestDataType.UUID),
+    ])
+
+    tester.populate_table('queueitem', [
+        ('queue_name', tester.TestDataType.String),
+        ('body', tester.TestDataType.JSON),
+        ('available_after', tester.TestDataType.DateTime),
+        ('available', tester.TestDataType.Boolean),
+        ('processing_expires', tester.TestDataType.DateTime),
+        ('retries_remaining', tester.TestDataType.Integer),
+    ])
+
+    tester.populate_table('servicekeyapproval', [
+        ('approver_id', tester.TestDataType.Foreign('user')),
+        ('approval_type', tester.TestDataType.String),
+        ('approved_date', tester.TestDataType.DateTime),
+        ('notes', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('servicekey', [
+        ('name', tester.TestDataType.String),
+        ('kid', tester.TestDataType.String),
+        ('service', tester.TestDataType.String),
+        ('jwk', tester.TestDataType.JSON),
+        ('metadata', tester.TestDataType.JSON),
+        ('created_date', tester.TestDataType.DateTime),
+        ('approval_id', tester.TestDataType.Foreign('servicekeyapproval')),
+    ])
+
+    tester.populate_table('label', [
+        ('uuid', tester.TestDataType.UUID),
+        ('key', tester.TestDataType.UTF8Char),
+        ('value', tester.TestDataType.JSON),
+        ('media_type_id', tester.TestDataType.Foreign('mediatype')),
+        ('source_type_id', tester.TestDataType.Foreign('labelsourcetype')),
+    ])
+
+    tester.populate_table('logentry', [
+        ('kind_id', tester.TestDataType.Foreign('logentrykind')),
+        ('account_id', tester.TestDataType.Foreign('user')),
+        ('performer_id', tester.TestDataType.Foreign('user')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('datetime', tester.TestDataType.DateTime),
+        ('ip', tester.TestDataType.String),
+        ('metadata_json', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('notification', [
+        ('uuid', tester.TestDataType.UUID),
+        ('kind_id', tester.TestDataType.Foreign('notificationkind')),
+        ('target_id', tester.TestDataType.Foreign('user')),
+        ('metadata_json', tester.TestDataType.JSON),
+        ('created', tester.TestDataType.DateTime),
+        ('dismissed', tester.TestDataType.Boolean),
+        ('lookup_path', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('oauthapplication', [
+        ('client_id', tester.TestDataType.String),
+        ('client_secret', tester.TestDataType.String),
+        ('redirect_uri', tester.TestDataType.String),
+        ('application_uri', tester.TestDataType.String),
+        ('organization_id', tester.TestDataType.Foreign('user')),
+        ('name', tester.TestDataType.String),
+        ('description', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('team', [
+        ('name', tester.TestDataType.String),
+        ('organization_id', tester.TestDataType.Foreign('user')),
+        ('role_id', tester.TestDataType.Foreign('teamrole')),
+        ('description', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('torrentinfo', [
+        ('storage_id', tester.TestDataType.Foreign('imagestorage')),
+        ('piece_length', tester.TestDataType.Integer),
+        ('pieces', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('userregion', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('location_id', tester.TestDataType.Foreign('imagestoragelocation')),
+    ])
+
+    tester.populate_table('accesstoken', [
+        ('friendly_name', tester.TestDataType.String),
+        ('code', tester.TestDataType.String),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('created', tester.TestDataType.DateTime),
+        ('role_id', tester.TestDataType.Foreign('role')),
+        ('temporary', tester.TestDataType.Boolean),
+        ('kind_id', tester.TestDataType.Foreign('accesstokenkind')),
+    ])
+
+    tester.populate_table('blobupload', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('uuid', tester.TestDataType.UUID),
+        ('byte_count', tester.TestDataType.Integer),
+        ('sha_state', tester.TestDataType.String),
+        ('location_id', tester.TestDataType.Foreign('imagestoragelocation')),
+        ('chunk_count', tester.TestDataType.Integer),
+        ('created', tester.TestDataType.DateTime),
+    ])
+
+    tester.populate_table('oauthaccesstoken', [
+        ('uuid', tester.TestDataType.UUID),
+        ('application_id', tester.TestDataType.Foreign('oauthapplication')),
+        ('authorized_user_id', tester.TestDataType.Foreign('user')),
+        ('scope', tester.TestDataType.String),
+        ('access_token', tester.TestDataType.String),
+        ('token_type', tester.TestDataType.String),
+        ('expires_at', tester.TestDataType.DateTime),
+        ('data', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('oauthauthorizationcode', [
+        ('application_id', tester.TestDataType.Foreign('oauthapplication')),
+        ('code', tester.TestDataType.String),
+        ('scope', tester.TestDataType.String),
+        ('data', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('permissionprototype', [
+        ('org_id', tester.TestDataType.Foreign('user')),
+        ('uuid', tester.TestDataType.UUID),
+        ('activating_user_id', tester.TestDataType.Foreign('user')),
+        ('delegate_user_id', tester.TestDataType.Foreign('user')),
+        ('role_id', tester.TestDataType.Foreign('role')),
+    ])
+
+    tester.populate_table('repositoryactioncount', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('count', tester.TestDataType.Integer),
+        ('date', tester.TestDataType.Date),
+    ])
+
+    tester.populate_table('repositoryauthorizedemail', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('email', tester.TestDataType.String),
+        ('code', tester.TestDataType.String),
+        ('confirmed', tester.TestDataType.Boolean),
+    ])
+
+    tester.populate_table('repositorynotification', [
+        ('uuid', tester.TestDataType.UUID),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('event_id', tester.TestDataType.Foreign('externalnotificationevent')),
+        ('method_id', tester.TestDataType.Foreign('externalnotificationmethod')),
+        ('title', tester.TestDataType.String),
+        ('config_json', tester.TestDataType.JSON),
+        ('event_config_json', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('repositorypermission', [
+        ('team_id', tester.TestDataType.Foreign('team')),
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('role_id', tester.TestDataType.Foreign('role')),
+    ])
+
+    tester.populate_table('star', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('created', tester.TestDataType.DateTime),
+    ])
+
+    tester.populate_table('teammember', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('team_id', tester.TestDataType.Foreign('team')),
+    ])
+
+    tester.populate_table('teammemberinvite', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('email', tester.TestDataType.String),
+        ('team_id', tester.TestDataType.Foreign('team')),
+        ('inviter_id', tester.TestDataType.Foreign('user')),
+        ('invite_token', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('derivedstorageforimage', [
+        ('source_image_id', tester.TestDataType.Foreign('image')),
+        ('derivative_id', tester.TestDataType.Foreign('imagestorage')),
+        ('transformation_id', tester.TestDataType.Foreign('imagestoragetransformation')),
+        ('uniqueness_hash', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('repositorybuildtrigger', [
+        ('uuid', tester.TestDataType.UUID),
+        ('service_id', tester.TestDataType.Foreign('buildtriggerservice')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('connected_user_id', tester.TestDataType.Foreign('user')),
+        ('auth_token', tester.TestDataType.String),
+        ('config', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('repositorytag', [
+        ('name', tester.TestDataType.String),
+        ('image_id', tester.TestDataType.Foreign('image')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('lifetime_start_ts', tester.TestDataType.Integer),
+        ('hidden', tester.TestDataType.Boolean),
+        ('reversion', tester.TestDataType.Boolean),
+    ])
+
+    tester.populate_table('repositorybuild', [
+        ('uuid', tester.TestDataType.UUID),
+        ('phase', tester.TestDataType.String),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('access_token_id', tester.TestDataType.Foreign('accesstoken')),
+        ('resource_key', tester.TestDataType.String),
+        ('job_config', tester.TestDataType.JSON),
+        ('started', tester.TestDataType.DateTime),
+        ('display_name', tester.TestDataType.JSON),
+        ('trigger_id', tester.TestDataType.Foreign('repositorybuildtrigger')),
+        ('logs_archived', tester.TestDataType.Boolean),
+    ])
+
+    tester.populate_table('tagmanifest', [
+        ('tag_id', tester.TestDataType.Foreign('repositorytag')),
+        ('digest', tester.TestDataType.String),
+        ('json_data', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('tagmanifestlabel', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('annotated_id', tester.TestDataType.Foreign('tagmanifest')),
+        ('label_id', tester.TestDataType.Foreign('label')),
+    ])
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
     op.drop_table('tagmanifestlabel')
     op.drop_table('tagmanifest')
     op.drop_table('repositorybuild')

data/migrations/versions/c3d4b7ebcdf7_backfill_repositorysearchscore_table.py

@@ -13,11 +13,11 @@ down_revision = 'f30984525c86'
 from alembic import op
 import sqlalchemy as sa
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # Add a 0 entry into the RepositorySearchScore table for each repository that isn't present
     conn = op.get_bind()
     conn.execute("insert into repositorysearchscore (repository_id, score) SELECT id, 0 FROM " +
                  "repository WHERE id not in (select repository_id from repositorysearchscore)")
 
-def downgrade(tables):
+def downgrade(tables, tester):
     pass

data/migrations/versions/c91c564aad34_drop_checksum_on_imagestorage.py

@@ -14,9 +14,9 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.drop_column('imagestorage', 'checksum')
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     op.add_column('imagestorage', sa.Column('checksum', sa.String(length=255), nullable=True))

data/migrations/versions/cbc8177760d9_add_user_location_field.py

@@ -15,9 +15,13 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 from util.migrate import UTF8CharField
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.add_column('user', sa.Column('location', UTF8CharField(length=255), nullable=True))
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'location', tester.TestDataType.UTF8Char)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     op.drop_column('user', 'location')

data/migrations/versions/d42c175b439a_backfill_state_id_and_make_it_unique.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # Backfill the queueitem table's state_id field with unique values for all entries which are
     # empty.
     conn = op.get_bind()
@@ -26,7 +26,7 @@ def upgrade(tables):
     # ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('queueitem_state_id', table_name='queueitem')
     op.create_index('queueitem_state_id', 'queueitem', ['state_id'], unique=False)

data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py

@@ -12,13 +12,13 @@ down_revision = 'dc4af11a5f90'
 
 from alembic import op
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.bulk_insert(tables.logentrykind, [
     {'name': 'change_tag_expiration'},
   ])
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   op.execute(tables
              .logentrykind
              .delete()

data/migrations/versions/dc4af11a5f90_add_notification_number_of_failures_.py

@@ -14,7 +14,7 @@ import sqlalchemy as sa
 from alembic import op
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.add_column('repositorynotification', sa.Column('number_of_failures',
                                                     sa.Integer(),
                                                     nullable=False,
@@ -23,8 +23,12 @@ def upgrade(tables):
     {'name': 'reset_repo_notification'},
   ])
 
+  # ### population of test data ### #
+  tester.populate_column('repositorynotification', 'number_of_failures', tester.TestDataType.Integer)
+  # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
   op.drop_column('repositorynotification', 'number_of_failures')
   op.execute(tables
              .logentrykind

data/migrations/versions/e2894a3a3c19_add_full_text_search_indexing_for_repo_.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     if op.get_bind().engine.name == 'postgresql':
       op.execute('CREATE EXTENSION IF NOT EXISTS pg_trgm')
 
@@ -24,7 +24,7 @@ def upgrade(tables):
     # ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('repository_name__fulltext', table_name='repository')
     op.drop_index('repository_description__fulltext', table_name='repository')

data/migrations/versions/ed01e313d3cb_add_trust_enabled_to_repository.py

@@ -13,7 +13,7 @@ down_revision = 'c3d4b7ebcdf7'
 from alembic import op
 import sqlalchemy as sa
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.add_column('repository', sa.Column('trust_enabled', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
     ### end Alembic commands ###
@@ -21,8 +21,12 @@ def upgrade(tables):
         {'name': 'change_repo_trust'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_column('repository', 'trust_enabled', tester.TestDataType.Boolean)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('repository', 'trust_enabled')
     ### end Alembic commands ###

data/migrations/versions/f30984525c86_add_repositorysearchscore_table.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table('repositorysearchscore',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -28,8 +28,16 @@ def upgrade(tables):
     op.create_index('repositorysearchscore_score', 'repositorysearchscore', ['score'], unique=False)
     ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('repositorysearchscore', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('score', tester.TestDataType.BigInteger),
+        ('last_updated', tester.TestDataType.DateTime),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('repositorysearchscore')
     ### end Alembic commands ###

data/migrations/versions/f5167870dd66_update_queue_item_table_indices.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_index('queueitem_processing_expires_available', 'queueitem', ['processing_expires', 'available'], unique=False)
     op.create_index('queueitem_pe_aafter_qname_rremaining_available', 'queueitem', ['processing_expires', 'available_after', 'queue_name', 'retries_remaining', 'available'], unique=False)
@@ -27,7 +27,7 @@ def upgrade(tables):
     ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_index('queueitem_retries_remaining', 'queueitem', ['retries_remaining'], unique=False)
     op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False)

data/migrations/versions/faf752bd2e0a_add_user_metadata_fields.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 
 from util.migrate import UTF8CharField
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('company', UTF8CharField(length=255), nullable=True))
     op.add_column('user', sa.Column('family_name', UTF8CharField(length=255), nullable=True))
@@ -28,8 +28,14 @@ def upgrade(tables):
         {'name':'enter_company'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'company', tester.TestDataType.UTF8Char)
+    tester.populate_column('user', 'family_name', tester.TestDataType.UTF8Char)
+    tester.populate_column('user', 'given_name', tester.TestDataType.UTF8Char)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('user', 'given_name')
     op.drop_column('user', 'family_name')

data/migrations/versions/fc47c1ec019f_add_state_id_field_to_queueitem.py

@@ -14,14 +14,18 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('queueitem', sa.Column('state_id', sa.String(length=255), nullable=False, server_default=''))
     op.create_index('queueitem_state_id', 'queueitem', ['state_id'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('queueitem', 'state_id', tester.TestDataType.String)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('queueitem_state_id', table_name='queueitem')
     op.drop_column('queueitem', 'state_id')

util/migrate/cleanup_old_robots.py

@@ -19,8 +19,11 @@ def cleanup_old_robots(page_size=50):
     for robot in list(User.select().where(User.robot == True).paginate(page_number, page_size)):
       found_bots = True
       logger.info("Checking robot %s (page %s)", robot.username, page_number)
-      namespace, _ = parse_robot_username(robot.username)
+      parsed = parse_robot_username(robot.username)
+      if parsed is None:
+        continue
 
+      namespace, _ = parsed
       if namespace in encountered_namespaces:
         if not encountered_namespaces[namespace]:
           logger.info('Marking %s to be deleted', robot.username)