Add support for populating test data during migration testing

This change ensures that the tables in the database during migration have at least one row of "real" data, which should help catch issues in the future where we forget to set column defaults and other such schema oversights that can only be truly tested with non-empty tables

Fixes https://jira.coreos.com/browse/QUAY-913

data/migrations/env.py

@@ -12,6 +12,8 @@ from urllib import unquote, quote
 from peewee import SqliteDatabase
 
 from data.database import all_models, db
+from data.migrations.tester import NoopTester, PopulateTestDataTester
+
 from app import app
 from data.model.sqlalchemybridge import gen_sqlalchemy_metadata
 from release import GIT_HEAD, REGION, SERVICE
@@ -39,6 +41,18 @@ tables = AttrDict(target_metadata.tables)
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
+def get_tester():
+    """ Returns the tester to use. We only return the tester that populates data
+        if the TEST_MIGRATE env var is set to `true` AND we make sure we're not
+        connecting to a production database.
+    """
+    if os.environ.get('TEST_MIGRATE', '') == 'true':
+        url = unquote(app.config['DB_URI'])
+        if url.find('.quay.io') < 0:
+            return PopulateTestDataTester()
+
+    return NoopTester()
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -55,7 +69,8 @@ def run_migrations_offline():
     context.configure(url=url, target_metadata=target_metadata, transactional_ddl=True)
 
     with context.begin_transaction():
-        context.run_migrations(tables=tables)
+        context.run_migrations(tables=tables, tester=get_tester())
+
 
 def run_migrations_online():
     """Run migrations in 'online' mode.
@@ -84,7 +99,7 @@ def run_migrations_online():
     try:
         with context.begin_transaction():
             try:
-                context.run_migrations(tables=tables)
+                context.run_migrations(tables=tables, tester=get_tester())
             except (CommandError, ResolutionError) as ex:
                 if 'No such revision' not in str(ex):
                     raise

data/migrations/migration.sh

@@ -8,7 +8,7 @@ PGSQL_CONFIG_OVERRIDE="{\"DB_URI\":\"postgresql://postgres@$DOCKER_IP/genschema\
 
 up_mysql() {
   # Run a SQL database on port 3306 inside of Docker.
-  docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql
+  docker run --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -d mysql:5.7
 
   echo 'Sleeping for 25...'
   sleep 25
@@ -18,8 +18,8 @@ up_mysql() {
 }
 
 down_mysql() {
-  docker kill mysql
-  docker rm -v mysql
+  docker kill mysql || true
+  docker rm -v mysql || true
 }
 
 up_mariadb() {
@@ -34,8 +34,8 @@ up_mariadb() {
 }
 
 down_mariadb() {
-  docker kill mariadb
-  docker rm -v mariadb
+  docker kill mariadb || true
+  docker rm -v mariadb || true
 }
 
 up_percona() {
@@ -50,8 +50,8 @@ up_percona() {
 }
 
 down_percona() {
-  docker kill percona
-  docker rm -v percona
+  docker kill percona || true
+  docker rm -v percona || true
 }
 
 up_postgres() {
@@ -67,8 +67,8 @@ up_postgres() {
 }
 
 down_postgres() {
-  docker kill postgres
-  docker rm -v postgres
+  docker kill postgres || true
+  docker rm -v postgres || true
 }
 
 gen_migrate() {
@@ -83,14 +83,19 @@ gen_migrate() {
 test_migrate() {
   # Generate a database with the schema as defined by the existing alembic model.
   echo '> Running upgrade'
-  QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head
+  TEST_MIGRATE=true QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic upgrade head
 
   # Downgrade to verify it works in both directions.
   echo '> Running downgrade'
   COUNT=`ls data/migrations/versions/*.py | wc -l | tr -d ' '`
-  QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT"
+  TEST_MIGRATE=true QUAY_OVERRIDE_CONFIG=$1 PYTHONPATH=. alembic downgrade "-$COUNT"
 }
 
+down_mysql
+down_postgres
+down_mariadb
+down_percona
+
 # Test (and generate, if requested) via MySQL.
 echo '> Starting MySQL'
 up_mysql

data/migrations/script.py.mako

@@ -14,9 +14,9 @@ from alembic import op
 import sqlalchemy as sa
 ${imports if imports else ""}
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ${upgrades if upgrades else "pass"}
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     ${downgrades if downgrades else "pass"}

data/migrations/tester.py

@@ -0,0 +1,122 @@
+import json
+import logging
+import uuid
+
+from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from six import add_metaclass
+
+from alembic import op
+from sqlalchemy import text
+
+from util.abchelpers import nooper
+
+logger = logging.getLogger(__name__)
+
+def escape_table_name(table_name):
+  if op.get_bind().engine.name == 'postgresql':
+    # Needed for the `user` table.
+    return '"%s"' % table_name
+
+  return table_name
+
+
+class DataTypes(object):
+  @staticmethod
+  def DateTime():
+    return datetime.now()
+
+  @staticmethod
+  def Date():
+    return datetime.now()
+
+  @staticmethod
+  def String():
+    return 'somestringvalue'
+
+  @staticmethod
+  def UTF8Char():
+    return 'some other value'
+
+  @staticmethod
+  def UUID():
+    return str(uuid.uuid4())
+
+  @staticmethod
+  def JSON():
+    return json.dumps(dict(foo='bar', baz='meh'))
+
+  @staticmethod
+  def Boolean():
+    if op.get_bind().engine.name == 'postgresql':
+      return True
+
+    return 1
+
+  @staticmethod
+  def BigInteger():
+    return 21474836470
+
+  @staticmethod
+  def Integer():
+    return 42
+
+  @staticmethod
+  def Foreign(table_name):
+    def get_index():
+      result = op.get_bind().execute("SELECT id FROM %s LIMIT 1" % escape_table_name(table_name))
+      try:
+        return list(result)[0][0]
+      except IndexError:
+        raise Exception('Could not find row for table %s' % table_name)
+      finally:
+        result.close()
+
+    return get_index
+
+
+@add_metaclass(ABCMeta)
+class MigrationTester(object):
+  """ Implements an interface for adding testing capabilities to the
+      data model migration system in Alembic.
+  """
+  TestDataType = DataTypes
+
+  @abstractmethod
+  def populate_table(self, table_name, fields):
+    """ Called to populate a table with the given fields filled in with testing data. """
+
+  @abstractmethod
+  def populate_column(self, table_name, col_name, field_type):
+    """ Called to populate a column in a table to be filled in with testing data. """
+
+
+@nooper
+class NoopTester(MigrationTester):
+  """ No-op version of the tester, designed for production workloads. """
+
+
+class PopulateTestDataTester(MigrationTester):
+  def populate_table(self, table_name, fields):
+    columns = {field_name: field_type() for field_name, field_type in fields}
+    field_name_vars = [':' + field_name for field_name, _ in fields]
+
+    if op.get_bind().engine.name == 'postgresql':
+      field_names = ["%s" % field_name for field_name, _ in fields]
+    else:
+      field_names = ["`%s`" % field_name for field_name, _ in fields]
+
+    table_name = escape_table_name(table_name)
+    query = text('INSERT INTO %s (%s) VALUES (%s)' % (table_name, ', '.join(field_names),
+                                                      ', '.join(field_name_vars)))
+    logger.info("Executing test query %s with values %s", query, columns.values())
+    op.get_bind().execute(query, **columns)
+
+  def populate_column(self, table_name, col_name, field_type):
+    col_value = field_type()
+    row_id = DataTypes.Foreign(table_name)()
+
+    table_name = escape_table_name(table_name)
+    update_text = text("UPDATE %s SET %s=:col_value where ID=:row_id" % (table_name, col_name))
+    logger.info("Executing test query %s with value %s on row %s", update_text, col_value, row_id)
+    op.get_bind().execute(update_text, col_value=col_value, row_id=row_id)

data/migrations/versions/0cf50323c78b_add_creation_date_to_user_table.py

@@ -14,13 +14,17 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('creation_date', sa.DateTime(), nullable=True))
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'creation_date', tester.TestDataType.DateTime)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('user', 'creation_date')
     # ### end Alembic commands ###

data/migrations/versions/152bb29a1bb3_add_maximum_build_queue_count_setting_.py

@@ -14,13 +14,17 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('maximum_queued_builds_count', sa.Integer(), nullable=True))
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'maximum_queued_builds_count', tester.TestDataType.Integer)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('user', 'maximum_queued_builds_count')
     # ### end Alembic commands ###

data/migrations/versions/152edccba18c_make_blodupload_byte_count_not_nullable.py

@@ -14,11 +14,11 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(),
                     nullable=False)
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(),
                     nullable=True)

data/migrations/versions/1783530bee68_add_logentry2_table_quay_io_only.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('logentry2',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -40,7 +40,7 @@ def upgrade(tables):
     # ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('logentry2')
     # ### end Alembic commands ###

data/migrations/versions/17aff2e1354e_add_automatic_disable_of_build_triggers.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('repositorybuildtrigger', sa.Column('successive_failure_count', sa.Integer(), server_default='0', nullable=False))
     op.add_column('repositorybuildtrigger', sa.Column('successive_internal_error_count', sa.Integer(), server_default='0', nullable=False))
@@ -28,7 +28,13 @@ def upgrade(tables):
         ],
     )
 
-def downgrade(tables):
+    # ### population of test data ### #
+    tester.populate_column('repositorybuildtrigger', 'successive_failure_count', tester.TestDataType.Integer)
+    tester.populate_column('repositorybuildtrigger', 'successive_internal_error_count', tester.TestDataType.Integer)
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('repositorybuildtrigger', 'successive_internal_error_count')
     op.drop_column('repositorybuildtrigger', 'successive_failure_count')

data/migrations/versions/224ce4c72c2f_add_last_accessed_field_to_user_table.py

@@ -14,14 +14,18 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('last_accessed', sa.DateTime(), nullable=True))
     op.create_index('user_last_accessed', 'user', ['last_accessed'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'last_accessed', tester.TestDataType.DateTime)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('user_last_accessed', table_name='user')
     op.drop_column('user', 'last_accessed')

data/migrations/versions/3e8cc74a1e7b_add_severity_and_media_type_to_global_.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('messages', sa.Column('media_type_id', sa.Integer(), nullable=False, server_default='1'))
     op.add_column('messages', sa.Column('severity', sa.String(length=255), nullable=False, server_default='info'))
@@ -33,8 +33,14 @@ def upgrade(tables):
                      {'name': 'text/markdown'},
                    ])
 
+    # ### population of test data ### #
+    tester.populate_column('messages', 'media_type_id', tester.TestDataType.Foreign('mediatype'))
+    tester.populate_column('messages', 'severity', lambda: 'info')
+    tester.populate_column('messages', 'uuid', tester.TestDataType.UUID)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_constraint(op.f('fk_messages_media_type_id_mediatype'), 'messages', type_='foreignkey')
     op.drop_index('messages_uuid', table_name='messages')

data/migrations/versions/45fd8b9869d4_add_notification_type.py

@@ -12,14 +12,14 @@ down_revision = '94836b099894'
 
 from alembic import op
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.bulk_insert(tables.notificationkind,
                  [
                    {'name': 'build_cancelled'},
                  ])
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   op.execute(tables
              .notificationkind
              .delete()

data/migrations/versions/53e2ac668296_remove_reference_to_subdir.py

@@ -33,11 +33,11 @@ def run_migration(migrate_function):
       log.warning("Failed to update build trigger %s with exception: ", trigger[0], e)
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   run_migration(delete_subdir)
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   run_migration(add_subdir)
 
 

data/migrations/versions/5b7503aada1b_cleanup_old_robots.py

@@ -15,9 +15,9 @@ import sqlalchemy as sa
 
 from util.migrate.cleanup_old_robots import cleanup_old_robots
 
-def upgrade(tables):
+def upgrade(tables, tester):
   cleanup_old_robots()
 
-def downgrade(tables):
+def downgrade(tables, tester):
   # Nothing to do.
   pass

data/migrations/versions/61cadbacb9fc_add_ability_for_build_triggers_to_be_.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('disablereason',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -40,8 +40,13 @@ def upgrade(tables):
     op.create_foreign_key(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', 'disablereason', ['disabled_reason_id'], ['id'])
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('repositorybuildtrigger', 'disabled_reason_id', tester.TestDataType.Foreign('disablereason'))
+    tester.populate_column('repositorybuildtrigger', 'enabled', tester.TestDataType.Boolean)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_constraint(op.f('fk_repositorybuildtrigger_disabled_reason_id_disablereason'), 'repositorybuildtrigger', type_='foreignkey')
     op.drop_index('repositorybuildtrigger_disabled_reason_id', table_name='repositorybuildtrigger')

data/migrations/versions/6c7014e84a5e_add_user_prompt_support.py

@@ -13,7 +13,7 @@ down_revision = 'c156deb8845d'
 from alembic import op
 import sqlalchemy as sa
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table('userpromptkind',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -39,8 +39,14 @@ def upgrade(tables):
         {'name':'confirm_username'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_table('userprompt', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('kind_id', tester.TestDataType.Foreign('userpromptkind')),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('userprompt')
     op.drop_table('userpromptkind')

data/migrations/versions/7367229b38d9_add_support_for_app_specific_tokens.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 from util.migrate import UTF8CharField
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('appspecificauthtoken',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -40,7 +40,20 @@ def upgrade(tables):
         {'name': 'revoke_app_specific_token'},
     ])
 
-def downgrade(tables):
+    # ### population of test data ### #
+    tester.populate_table('appspecificauthtoken', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('uuid', tester.TestDataType.UUID),
+        ('title', tester.TestDataType.UTF8Char),
+        ('token_code', tester.TestDataType.String),
+        ('created', tester.TestDataType.DateTime),
+        ('expiration', tester.TestDataType.DateTime),
+        ('last_accessed', tester.TestDataType.DateTime),
+    ])
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('appspecificauthtoken')
     # ### end Alembic commands ###

data/migrations/versions/7a525c68eb13_add_oci_app_models.py

@@ -16,7 +16,7 @@ from sqlalchemy.dialects import mysql
 from util.migrate import UTF8LongText, UTF8CharField
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.create_table(
     'tagkind',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -307,7 +307,7 @@ def upgrade(tables):
   )
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   op.drop_table('manifestlayerscan')
   op.drop_table('manifestlayerdockerv1')
   op.drop_table('tag')

data/migrations/versions/87fbbc224f10_add_disabled_datetime_to_trigger.py

@@ -14,14 +14,18 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('repositorybuildtrigger', sa.Column('disabled_datetime', sa.DateTime(), nullable=True))
     op.create_index('repositorybuildtrigger_disabled_datetime', 'repositorybuildtrigger', ['disabled_datetime'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('repositorybuildtrigger', 'disabled_datetime', tester.TestDataType.DateTime)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('repositorybuildtrigger_disabled_datetime', table_name='repositorybuildtrigger')
     op.drop_column('repositorybuildtrigger', 'disabled_datetime')

data/migrations/versions/94836b099894_create_new_notification_type.py

@@ -13,14 +13,14 @@ down_revision = 'faf752bd2e0a'
 from alembic import op
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.bulk_insert(tables.externalnotificationevent,
                    [
                        {'name': 'build_cancelled'},
                    ])
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     op.execute(tables
                .externalnotificationevent
                .delete()

data/migrations/versions/a6c463dfb9fe_back_fill_build_expand_config.py

@@ -22,7 +22,7 @@ from alembic import op
 class RepositoryBuildTrigger(BaseModel):
   config = TextField(default='{}')
 
-def upgrade(tables):
+def upgrade(tables, tester):
   repostioryBuildTriggers = RepositoryBuildTrigger.select()
   for repositoryBuildTrigger in repostioryBuildTriggers:
     config = json.loads(repositoryBuildTrigger.config)
@@ -30,7 +30,7 @@ def upgrade(tables):
     repositoryBuildTrigger.save()
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   repostioryBuildTriggers = RepositoryBuildTrigger.select()
   for repositoryBuildTrigger in repostioryBuildTriggers:
     config = json.loads(repositoryBuildTrigger.config)

data/migrations/versions/b4c2d45bc132_add_deleted_namespace_table.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('deletednamespace',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -32,8 +32,19 @@ def upgrade(tables):
     op.create_index('deletednamespace_queue_id', 'deletednamespace', ['queue_id'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('deletednamespace', [
+        ('namespace_id', tester.TestDataType.Foreign('user')),
+        ('marked', tester.TestDataType.DateTime),
+        ('original_username', tester.TestDataType.UTF8Char),
+        ('original_email', tester.TestDataType.String),
+        ('queue_id', tester.TestDataType.Foreign('queueitem')),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('deletednamespace')
     # ### end Alembic commands ###

data/migrations/versions/b4df55dea4b3_add_repository_kind.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.create_table(
     'repositorykind',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -36,8 +36,12 @@ def upgrade(tables):
   op.create_index('repository_kind_id', 'repository', ['kind_id'], unique=False)
   op.create_foreign_key(op.f('fk_repository_kind_id_repositorykind'), 'repository', 'repositorykind', ['kind_id'], ['id'])
 
+  # ### population of test data ### #
+  tester.populate_column('repository', 'kind_id', tester.TestDataType.Foreign('repositorykind'))
+  # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
   op.drop_constraint(op.f('fk_repository_kind_id_repositorykind'), 'repository', type_='foreignkey')
   op.drop_index('repository_kind_id', table_name='repository')
   op.drop_column(u'repository', 'kind_id')

data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from util.migrate import UTF8CharField
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('robotaccountmetadata',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -28,8 +28,16 @@ def upgrade(tables):
     op.create_index('robotaccountmetadata_robot_account_id', 'robotaccountmetadata', ['robot_account_id'], unique=True)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('robotaccountmetadata', [
+        ('robot_account_id', tester.TestDataType.Foreign('user')),        
+        ('description', tester.TestDataType.UTF8Char),        
+        ('unstructured_json', tester.TestDataType.JSON),        
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('robotaccountmetadata')
     # ### end Alembic commands ###

data/migrations/versions/b8ae68ad3e52_change_blobupload_fields_to_bigintegers_.py

@@ -14,11 +14,21 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.Integer(), type_=sa.BigInteger())
     op.alter_column('blobupload', 'uncompressed_byte_count', existing_type=sa.Integer(), type_=sa.BigInteger())
 
+    # ### population of test data ### #
+    tester.populate_column('blobupload', 'byte_count', tester.TestDataType.BigInteger)
+    tester.populate_column('blobupload', 'uncompressed_byte_count', tester.TestDataType.BigInteger)
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
+    # ### population of test data ### #
+    tester.populate_column('blobupload', 'byte_count', tester.TestDataType.Integer)
+    tester.populate_column('blobupload', 'uncompressed_byte_count', tester.TestDataType.Integer)
+    # ### end population of test data ### #
 
-def downgrade(tables):
     op.alter_column('blobupload', 'byte_count', existing_type=sa.BigInteger(), type_=sa.Integer())
     op.alter_column('blobupload', 'uncompressed_byte_count', existing_type=sa.BigInteger(), type_=sa.Integer())

data/migrations/versions/be8d1c402ce0_add_teamsync_table.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from util.migrate import UTF8LongText
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table('teamsync',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -32,8 +32,18 @@ def upgrade(tables):
     op.create_index('teamsync_team_id', 'teamsync', ['team_id'], unique=True)
     ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('teamsync', [
+        ('team_id', tester.TestDataType.Foreign('team')),
+        ('transaction_id', tester.TestDataType.String),
+        ('last_updated', tester.TestDataType.DateTime),
+        ('service_id', tester.TestDataType.Foreign('loginservice')),
+        ('config', tester.TestDataType.JSON),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('teamsync')
     ### end Alembic commands ###

data/migrations/versions/c156deb8845d_reset_our_migrations_with_a_required_.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 from util.migrate import UTF8LongText, UTF8CharField
 from datetime import datetime
 
-def upgrade(tables):
+def upgrade(tables, tester):
     now = datetime.now().strftime("'%Y-%m-%d %H:%M:%S'")
 
     op.create_table('accesstokenkind',
@@ -895,8 +895,305 @@ def upgrade(tables):
         {'name':'private'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_table('user', [
+        ('uuid', tester.TestDataType.UUID),
+        ('username', tester.TestDataType.String),
+        ('password_hash', tester.TestDataType.String),
+        ('email', tester.TestDataType.String),
+        ('verified', tester.TestDataType.Boolean),
+        ('organization', tester.TestDataType.Boolean),
+        ('robot', tester.TestDataType.Boolean),
+        ('invoice_email', tester.TestDataType.Boolean),
+        ('invalid_login_attempts', tester.TestDataType.Integer),
+        ('last_invalid_login', tester.TestDataType.DateTime),
+        ('removed_tag_expiration_s', tester.TestDataType.Integer),
+        ('enabled', tester.TestDataType.Boolean),
+        ('invoice_email_address', tester.TestDataType.String),
+    ])
 
-def downgrade(tables):
+    tester.populate_table('repository', [
+        ('namespace_user_id', tester.TestDataType.Foreign('user')),
+        ('name', tester.TestDataType.String),
+        ('visibility_id', tester.TestDataType.Foreign('visibility')),
+        ('description', tester.TestDataType.String),
+        ('badge_token', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('emailconfirmation', [
+        ('code', tester.TestDataType.String),
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('pw_reset', tester.TestDataType.Boolean),
+        ('email_confirm', tester.TestDataType.Boolean),
+        ('created', tester.TestDataType.DateTime),
+    ])
+
+    tester.populate_table('federatedlogin', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('service_id', tester.TestDataType.Foreign('loginservice')),
+        ('service_ident', tester.TestDataType.String),
+        ('metadata_json', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('imagestorage', [
+        ('uuid', tester.TestDataType.UUID),
+        ('checksum', tester.TestDataType.String),
+        ('image_size', tester.TestDataType.BigInteger),
+        ('uncompressed_size', tester.TestDataType.BigInteger),
+        ('uploading', tester.TestDataType.Boolean),
+        ('cas_path', tester.TestDataType.Boolean),
+        ('content_checksum', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('image', [
+        ('docker_image_id', tester.TestDataType.UUID),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('ancestors', tester.TestDataType.String),
+        ('storage_id', tester.TestDataType.Foreign('imagestorage')),
+        ('security_indexed', tester.TestDataType.Boolean),
+        ('security_indexed_engine', tester.TestDataType.Integer),
+    ])
+
+    tester.populate_table('imagestorageplacement', [
+        ('storage_id', tester.TestDataType.Foreign('imagestorage')),
+        ('location_id', tester.TestDataType.Foreign('imagestoragelocation')),
+    ])
+
+    tester.populate_table('messages', [
+        ('content', tester.TestDataType.String),
+        ('uuid', tester.TestDataType.UUID),
+    ])
+
+    tester.populate_table('queueitem', [
+        ('queue_name', tester.TestDataType.String),
+        ('body', tester.TestDataType.JSON),
+        ('available_after', tester.TestDataType.DateTime),
+        ('available', tester.TestDataType.Boolean),
+        ('processing_expires', tester.TestDataType.DateTime),
+        ('retries_remaining', tester.TestDataType.Integer),
+    ])
+
+    tester.populate_table('servicekeyapproval', [
+        ('approver_id', tester.TestDataType.Foreign('user')),
+        ('approval_type', tester.TestDataType.String),
+        ('approved_date', tester.TestDataType.DateTime),
+        ('notes', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('servicekey', [
+        ('name', tester.TestDataType.String),
+        ('kid', tester.TestDataType.String),
+        ('service', tester.TestDataType.String),
+        ('jwk', tester.TestDataType.JSON),
+        ('metadata', tester.TestDataType.JSON),
+        ('created_date', tester.TestDataType.DateTime),
+        ('approval_id', tester.TestDataType.Foreign('servicekeyapproval')),
+    ])
+
+    tester.populate_table('label', [
+        ('uuid', tester.TestDataType.UUID),
+        ('key', tester.TestDataType.UTF8Char),
+        ('value', tester.TestDataType.JSON),
+        ('media_type_id', tester.TestDataType.Foreign('mediatype')),
+        ('source_type_id', tester.TestDataType.Foreign('labelsourcetype')),
+    ])
+
+    tester.populate_table('logentry', [
+        ('kind_id', tester.TestDataType.Foreign('logentrykind')),
+        ('account_id', tester.TestDataType.Foreign('user')),
+        ('performer_id', tester.TestDataType.Foreign('user')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('datetime', tester.TestDataType.DateTime),
+        ('ip', tester.TestDataType.String),
+        ('metadata_json', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('notification', [
+        ('uuid', tester.TestDataType.UUID),
+        ('kind_id', tester.TestDataType.Foreign('notificationkind')),
+        ('target_id', tester.TestDataType.Foreign('user')),
+        ('metadata_json', tester.TestDataType.JSON),
+        ('created', tester.TestDataType.DateTime),
+        ('dismissed', tester.TestDataType.Boolean),
+        ('lookup_path', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('oauthapplication', [
+        ('client_id', tester.TestDataType.String),
+        ('client_secret', tester.TestDataType.String),
+        ('redirect_uri', tester.TestDataType.String),
+        ('application_uri', tester.TestDataType.String),
+        ('organization_id', tester.TestDataType.Foreign('user')),
+        ('name', tester.TestDataType.String),
+        ('description', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('team', [
+        ('name', tester.TestDataType.String),
+        ('organization_id', tester.TestDataType.Foreign('user')),
+        ('role_id', tester.TestDataType.Foreign('teamrole')),
+        ('description', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('torrentinfo', [
+        ('storage_id', tester.TestDataType.Foreign('imagestorage')),
+        ('piece_length', tester.TestDataType.Integer),
+        ('pieces', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('userregion', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('location_id', tester.TestDataType.Foreign('imagestoragelocation')),
+    ])
+
+    tester.populate_table('accesstoken', [
+        ('friendly_name', tester.TestDataType.String),
+        ('code', tester.TestDataType.String),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('created', tester.TestDataType.DateTime),
+        ('role_id', tester.TestDataType.Foreign('role')),
+        ('temporary', tester.TestDataType.Boolean),
+        ('kind_id', tester.TestDataType.Foreign('accesstokenkind')),
+    ])
+
+    tester.populate_table('blobupload', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('uuid', tester.TestDataType.UUID),
+        ('byte_count', tester.TestDataType.Integer),
+        ('sha_state', tester.TestDataType.String),
+        ('location_id', tester.TestDataType.Foreign('imagestoragelocation')),
+        ('chunk_count', tester.TestDataType.Integer),
+        ('created', tester.TestDataType.DateTime),
+    ])
+
+    tester.populate_table('oauthaccesstoken', [
+        ('uuid', tester.TestDataType.UUID),
+        ('application_id', tester.TestDataType.Foreign('oauthapplication')),
+        ('authorized_user_id', tester.TestDataType.Foreign('user')),
+        ('scope', tester.TestDataType.String),
+        ('access_token', tester.TestDataType.String),
+        ('token_type', tester.TestDataType.String),
+        ('expires_at', tester.TestDataType.DateTime),
+        ('data', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('oauthauthorizationcode', [
+        ('application_id', tester.TestDataType.Foreign('oauthapplication')),
+        ('code', tester.TestDataType.String),
+        ('scope', tester.TestDataType.String),
+        ('data', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('permissionprototype', [
+        ('org_id', tester.TestDataType.Foreign('user')),
+        ('uuid', tester.TestDataType.UUID),
+        ('activating_user_id', tester.TestDataType.Foreign('user')),
+        ('delegate_user_id', tester.TestDataType.Foreign('user')),
+        ('role_id', tester.TestDataType.Foreign('role')),
+    ])
+
+    tester.populate_table('repositoryactioncount', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('count', tester.TestDataType.Integer),
+        ('date', tester.TestDataType.Date),
+    ])
+
+    tester.populate_table('repositoryauthorizedemail', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('email', tester.TestDataType.String),
+        ('code', tester.TestDataType.String),
+        ('confirmed', tester.TestDataType.Boolean),
+    ])
+
+    tester.populate_table('repositorynotification', [
+        ('uuid', tester.TestDataType.UUID),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('event_id', tester.TestDataType.Foreign('externalnotificationevent')),
+        ('method_id', tester.TestDataType.Foreign('externalnotificationmethod')),
+        ('title', tester.TestDataType.String),
+        ('config_json', tester.TestDataType.JSON),
+        ('event_config_json', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('repositorypermission', [
+        ('team_id', tester.TestDataType.Foreign('team')),
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('role_id', tester.TestDataType.Foreign('role')),
+    ])
+
+    tester.populate_table('star', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('created', tester.TestDataType.DateTime),
+    ])
+
+    tester.populate_table('teammember', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('team_id', tester.TestDataType.Foreign('team')),
+    ])
+
+    tester.populate_table('teammemberinvite', [
+        ('user_id', tester.TestDataType.Foreign('user')),
+        ('email', tester.TestDataType.String),
+        ('team_id', tester.TestDataType.Foreign('team')),
+        ('inviter_id', tester.TestDataType.Foreign('user')),
+        ('invite_token', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('derivedstorageforimage', [
+        ('source_image_id', tester.TestDataType.Foreign('image')),
+        ('derivative_id', tester.TestDataType.Foreign('imagestorage')),
+        ('transformation_id', tester.TestDataType.Foreign('imagestoragetransformation')),
+        ('uniqueness_hash', tester.TestDataType.String),
+    ])
+
+    tester.populate_table('repositorybuildtrigger', [
+        ('uuid', tester.TestDataType.UUID),
+        ('service_id', tester.TestDataType.Foreign('buildtriggerservice')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('connected_user_id', tester.TestDataType.Foreign('user')),
+        ('auth_token', tester.TestDataType.String),
+        ('config', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('repositorytag', [
+        ('name', tester.TestDataType.String),
+        ('image_id', tester.TestDataType.Foreign('image')),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('lifetime_start_ts', tester.TestDataType.Integer),
+        ('hidden', tester.TestDataType.Boolean),
+        ('reversion', tester.TestDataType.Boolean),
+    ])
+
+    tester.populate_table('repositorybuild', [
+        ('uuid', tester.TestDataType.UUID),
+        ('phase', tester.TestDataType.String),
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('access_token_id', tester.TestDataType.Foreign('accesstoken')),
+        ('resource_key', tester.TestDataType.String),
+        ('job_config', tester.TestDataType.JSON),
+        ('started', tester.TestDataType.DateTime),
+        ('display_name', tester.TestDataType.JSON),
+        ('trigger_id', tester.TestDataType.Foreign('repositorybuildtrigger')),
+        ('logs_archived', tester.TestDataType.Boolean),
+    ])
+
+    tester.populate_table('tagmanifest', [
+        ('tag_id', tester.TestDataType.Foreign('repositorytag')),
+        ('digest', tester.TestDataType.String),
+        ('json_data', tester.TestDataType.JSON),
+    ])
+
+    tester.populate_table('tagmanifestlabel', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('annotated_id', tester.TestDataType.Foreign('tagmanifest')),
+        ('label_id', tester.TestDataType.Foreign('label')),
+    ])
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
     op.drop_table('tagmanifestlabel')
     op.drop_table('tagmanifest')
     op.drop_table('repositorybuild')

data/migrations/versions/c3d4b7ebcdf7_backfill_repositorysearchscore_table.py

@@ -13,11 +13,11 @@ down_revision = 'f30984525c86'
 from alembic import op
 import sqlalchemy as sa
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # Add a 0 entry into the RepositorySearchScore table for each repository that isn't present
     conn = op.get_bind()
     conn.execute("insert into repositorysearchscore (repository_id, score) SELECT id, 0 FROM " +
                  "repository WHERE id not in (select repository_id from repositorysearchscore)")
 
-def downgrade(tables):
+def downgrade(tables, tester):
     pass

data/migrations/versions/c91c564aad34_drop_checksum_on_imagestorage.py

@@ -14,9 +14,9 @@ from alembic import op
 import sqlalchemy as sa
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.drop_column('imagestorage', 'checksum')
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     op.add_column('imagestorage', sa.Column('checksum', sa.String(length=255), nullable=True))

data/migrations/versions/cbc8177760d9_add_user_location_field.py

@@ -15,9 +15,13 @@ import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 from util.migrate import UTF8CharField
 
-def upgrade(tables):
+def upgrade(tables, tester):
     op.add_column('user', sa.Column('location', UTF8CharField(length=255), nullable=True))
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'location', tester.TestDataType.UTF8Char)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     op.drop_column('user', 'location')

data/migrations/versions/d42c175b439a_backfill_state_id_and_make_it_unique.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # Backfill the queueitem table's state_id field with unique values for all entries which are
     # empty.
     conn = op.get_bind()
@@ -26,7 +26,7 @@ def upgrade(tables):
     # ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('queueitem_state_id', table_name='queueitem')
     op.create_index('queueitem_state_id', 'queueitem', ['state_id'], unique=False)

data/migrations/versions/d8989249f8f6_add_change_tag_expiration_log_type.py

@@ -12,13 +12,13 @@ down_revision = 'dc4af11a5f90'
 
 from alembic import op
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.bulk_insert(tables.logentrykind, [
     {'name': 'change_tag_expiration'},
   ])
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
   op.execute(tables
              .logentrykind
              .delete()

data/migrations/versions/dc4af11a5f90_add_notification_number_of_failures_.py

@@ -14,7 +14,7 @@ import sqlalchemy as sa
 from alembic import op
 
 
-def upgrade(tables):
+def upgrade(tables, tester):
   op.add_column('repositorynotification', sa.Column('number_of_failures',
                                                     sa.Integer(),
                                                     nullable=False,
@@ -23,8 +23,12 @@ def upgrade(tables):
     {'name': 'reset_repo_notification'},
   ])
 
+  # ### population of test data ### #
+  tester.populate_column('repositorynotification', 'number_of_failures', tester.TestDataType.Integer)
+  # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
   op.drop_column('repositorynotification', 'number_of_failures')
   op.execute(tables
              .logentrykind

data/migrations/versions/e2894a3a3c19_add_full_text_search_indexing_for_repo_.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     if op.get_bind().engine.name == 'postgresql':
       op.execute('CREATE EXTENSION IF NOT EXISTS pg_trgm')
 
@@ -24,7 +24,7 @@ def upgrade(tables):
     # ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('repository_name__fulltext', table_name='repository')
     op.drop_index('repository_description__fulltext', table_name='repository')

data/migrations/versions/ed01e313d3cb_add_trust_enabled_to_repository.py

@@ -13,7 +13,7 @@ down_revision = 'c3d4b7ebcdf7'
 from alembic import op
 import sqlalchemy as sa
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.add_column('repository', sa.Column('trust_enabled', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
     ### end Alembic commands ###
@@ -21,8 +21,12 @@ def upgrade(tables):
         {'name': 'change_repo_trust'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_column('repository', 'trust_enabled', tester.TestDataType.Boolean)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('repository', 'trust_enabled')
     ### end Alembic commands ###

data/migrations/versions/f30984525c86_add_repositorysearchscore_table.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_table('repositorysearchscore',
     sa.Column('id', sa.Integer(), nullable=False),
@@ -28,8 +28,16 @@ def upgrade(tables):
     op.create_index('repositorysearchscore_score', 'repositorysearchscore', ['score'], unique=False)
     ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_table('repositorysearchscore', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('score', tester.TestDataType.BigInteger),
+        ('last_updated', tester.TestDataType.DateTime),
+    ])
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_table('repositorysearchscore')
     ### end Alembic commands ###

data/migrations/versions/f5167870dd66_update_queue_item_table_indices.py

@@ -14,7 +14,7 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_index('queueitem_processing_expires_available', 'queueitem', ['processing_expires', 'available'], unique=False)
     op.create_index('queueitem_pe_aafter_qname_rremaining_available', 'queueitem', ['processing_expires', 'available_after', 'queue_name', 'retries_remaining', 'available'], unique=False)
@@ -27,7 +27,7 @@ def upgrade(tables):
     ### end Alembic commands ###
 
 
-def downgrade(tables):
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.create_index('queueitem_retries_remaining', 'queueitem', ['retries_remaining'], unique=False)
     op.create_index('queueitem_processing_expires', 'queueitem', ['processing_expires'], unique=False)

data/migrations/versions/faf752bd2e0a_add_user_metadata_fields.py

@@ -15,7 +15,7 @@ import sqlalchemy as sa
 
 from util.migrate import UTF8CharField
 
-def upgrade(tables):
+def upgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.add_column('user', sa.Column('company', UTF8CharField(length=255), nullable=True))
     op.add_column('user', sa.Column('family_name', UTF8CharField(length=255), nullable=True))
@@ -28,8 +28,14 @@ def upgrade(tables):
         {'name':'enter_company'},
     ])
 
+    # ### population of test data ### #
+    tester.populate_column('user', 'company', tester.TestDataType.UTF8Char)
+    tester.populate_column('user', 'family_name', tester.TestDataType.UTF8Char)
+    tester.populate_column('user', 'given_name', tester.TestDataType.UTF8Char)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     ### commands auto generated by Alembic - please adjust! ###
     op.drop_column('user', 'given_name')
     op.drop_column('user', 'family_name')

data/migrations/versions/fc47c1ec019f_add_state_id_field_to_queueitem.py

@@ -14,14 +14,18 @@ from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
 
-def upgrade(tables):
+def upgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column('queueitem', sa.Column('state_id', sa.String(length=255), nullable=False, server_default=''))
     op.create_index('queueitem_state_id', 'queueitem', ['state_id'], unique=False)
     # ### end Alembic commands ###
 
+    # ### population of test data ### #
+    tester.populate_column('queueitem', 'state_id', tester.TestDataType.String)
+    # ### end population of test data ### #
 
-def downgrade(tables):
+
+def downgrade(tables, tester):
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index('queueitem_state_id', table_name='queueitem')
     op.drop_column('queueitem', 'state_id')

util/migrate/cleanup_old_robots.py

@@ -19,8 +19,11 @@ def cleanup_old_robots(page_size=50):
     for robot in list(User.select().where(User.robot == True).paginate(page_number, page_size)):
       found_bots = True
       logger.info("Checking robot %s (page %s)", robot.username, page_number)
-      namespace, _ = parse_robot_username(robot.username)
+      parsed = parse_robot_username(robot.username)
+      if parsed is None:
+        continue
 
+      namespace, _ = parsed
       if namespace in encountered_namespaces:
         if not encountered_namespaces[namespace]:
           logger.info('Marking %s to be deleted', robot.username)