Add Tag, TagKind and ManifestChild tables in prep for new data model

data/database.py

@@ -1385,6 +1385,58 @@ class Manifest(BaseModel):
     )
 
 
+class TagKind(BaseModel):
+  """ TagKind describes the various kinds of tags that can be found in the registry.
+  """
+  name = CharField(index=True, unique=True)
+
+
+class Tag(BaseModel):
+  """ Tag represents a user-facing alias for referencing a Manifest or as an alias to another tag.
+  """
+  name = CharField()
+  repository = ForeignKeyField(Repository)
+  manifest = ForeignKeyField(Manifest, null=True)
+  lifetime_start_ms = BigIntegerField(default=get_epoch_timestamp_ms)
+  lifetime_end_ms = BigIntegerField(null=True, index=True)
+  hidden = BooleanField(default=False)
+  reverted = BooleanField(default=False)
+  tag_kind = EnumField(TagKind)
+  linked_tag = ForeignKeyField('self', null=True, backref='tag_parents')
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('repository', 'name'), False),
+      (('repository', 'name', 'hidden'), False),
+      (('repository', 'name', 'tag_kind'), False),
+
+      # This unique index prevents deadlocks when concurrently moving and deleting tags
+      (('repository', 'name', 'lifetime_end_ms'), True),
+    )
+
+
+class ManifestChild(BaseModel):
+  """ ManifestChild represents a relationship between a manifest and its child manifest(s).
+      Multiple manifests can share the same children. Note that since Manifests are stored
+      per-repository, the repository here is a bit redundant, but we do so to make cleanup easier.
+  """
+  repository = ForeignKeyField(Repository)
+  manifest = ForeignKeyField(Manifest)
+  child_manifest = ForeignKeyField(Manifest)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('repository', 'manifest'), False),
+      (('repository', 'child_manifest'), False),
+      (('repository', 'manifest', 'child_manifest'), False),
+      (('manifest', 'child_manifest'), True),
+    )
+
+
 class ManifestLabel(BaseModel):
   """ ManifestLabel represents a label applied to a Manifest, within a repository.
       Note that since Manifests are stored per-repository, the repository here is
@@ -1470,7 +1522,8 @@ class TagManifestLabelMap(BaseModel):
 appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
                     ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
                     ApprBlobPlacement])
-v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage])
+v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage, TagKind,
+                   ManifestChild, Tag])
 transition_classes = set([TagManifestToManifest, TagManifestLabelMap])
 
 is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel

data/migrations/versions/10f45ee2310b_add_tag_tagkind_and_manifestchild_tables.py

@@ -0,0 +1,97 @@
+"""Add Tag, TagKind and ManifestChild tables
+
+Revision ID: 10f45ee2310b
+Revises: 13411de1c0ff
+Create Date: 2018-10-29 15:22:53.552216
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '10f45ee2310b'
+down_revision = '13411de1c0ff'
+
+from alembic import op
+import sqlalchemy as sa
+from util.migrate import UTF8CharField
+
+def upgrade(tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('tagkind',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_tagkind'))
+    )
+    op.create_index('tagkind_name', 'tagkind', ['name'], unique=True)
+    op.create_table('manifestchild',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('repository_id', sa.Integer(), nullable=False),
+    sa.Column('manifest_id', sa.Integer(), nullable=False),
+    sa.Column('child_manifest_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['child_manifest_id'], ['manifest.id'], name=op.f('fk_manifestchild_child_manifest_id_manifest')),
+    sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestchild_manifest_id_manifest')),
+    sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_manifestchild_repository_id_repository')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestchild'))
+    )
+    op.create_index('manifestchild_child_manifest_id', 'manifestchild', ['child_manifest_id'], unique=False)
+    op.create_index('manifestchild_manifest_id', 'manifestchild', ['manifest_id'], unique=False)
+    op.create_index('manifestchild_manifest_id_child_manifest_id', 'manifestchild', ['manifest_id', 'child_manifest_id'], unique=True)
+    op.create_index('manifestchild_repository_id', 'manifestchild', ['repository_id'], unique=False)
+    op.create_index('manifestchild_repository_id_child_manifest_id', 'manifestchild', ['repository_id', 'child_manifest_id'], unique=False)
+    op.create_index('manifestchild_repository_id_manifest_id', 'manifestchild', ['repository_id', 'manifest_id'], unique=False)
+    op.create_index('manifestchild_repository_id_manifest_id_child_manifest_id', 'manifestchild', ['repository_id', 'manifest_id', 'child_manifest_id'], unique=False)
+    op.create_table('tag',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', UTF8CharField(length=255), nullable=False),
+    sa.Column('repository_id', sa.Integer(), nullable=False),
+    sa.Column('manifest_id', sa.Integer(), nullable=True),
+    sa.Column('lifetime_start_ms', sa.BigInteger(), nullable=False),
+    sa.Column('lifetime_end_ms', sa.BigInteger(), nullable=True),
+    sa.Column('hidden', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()),
+    sa.Column('reverted', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()),
+    sa.Column('tag_kind_id', sa.Integer(), nullable=False),
+    sa.Column('linked_tag_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['linked_tag_id'], ['tag.id'], name=op.f('fk_tag_linked_tag_id_tag')),
+    sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_tag_manifest_id_manifest')),
+    sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tag_repository_id_repository')),
+    sa.ForeignKeyConstraint(['tag_kind_id'], ['tagkind.id'], name=op.f('fk_tag_tag_kind_id_tagkind')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
+    )
+    op.create_index('tag_lifetime_end_ms', 'tag', ['lifetime_end_ms'], unique=False)
+    op.create_index('tag_linked_tag_id', 'tag', ['linked_tag_id'], unique=False)
+    op.create_index('tag_manifest_id', 'tag', ['manifest_id'], unique=False)
+    op.create_index('tag_repository_id', 'tag', ['repository_id'], unique=False)
+    op.create_index('tag_repository_id_name', 'tag', ['repository_id', 'name'], unique=False)
+    op.create_index('tag_repository_id_name_hidden', 'tag', ['repository_id', 'name', 'hidden'], unique=False)
+    op.create_index('tag_repository_id_name_lifetime_end_ms', 'tag', ['repository_id', 'name', 'lifetime_end_ms'], unique=True)
+    op.create_index('tag_repository_id_name_tag_kind_id', 'tag', ['repository_id', 'name', 'tag_kind_id'], unique=False)
+    op.create_index('tag_tag_kind_id', 'tag', ['tag_kind_id'], unique=False)
+    # ### end Alembic commands ###
+
+    op.bulk_insert(tables.tagkind,
+    [
+        {'name': 'tag'},
+    ])
+
+    # ### population of test data ### #
+    tester.populate_table('tag', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('tag_kind_id', tester.TestDataType.Foreign('tagkind')),
+        ('name', tester.TestDataType.String),
+        ('manifest_id', tester.TestDataType.Foreign('manifest')),
+        ('lifetime_start_ms', tester.TestDataType.BigInteger),
+    ])
+
+    tester.populate_table('manifestchild', [
+        ('repository_id', tester.TestDataType.Foreign('repository')),
+        ('manifest_id', tester.TestDataType.Foreign('manifest')),
+        ('child_manifest_id', tester.TestDataType.Foreign('manifest')),
+    ])
+    # ### end population of test data ### #
+
+
+def downgrade(tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('tag')
+    op.drop_table('manifestchild')
+    op.drop_table('tagkind')
+    # ### end Alembic commands ###

initdb.py

@@ -20,7 +20,8 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ
                            QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
                            ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
                            RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes,
-                           ApprTagKind, ApprBlobPlacementLocation, Repository)
+                           ApprTagKind, ApprBlobPlacementLocation, Repository, Tag, TagKind,
+                           ManifestChild)
 from data import model
 from data.queue import WorkQueue
 from data.registry_model import registry_model
@@ -450,6 +451,8 @@ def initialize_database():
   DisableReason.create(name='successive_build_failures')
   DisableReason.create(name='successive_build_internal_errors')
 
+  TagKind.create(name='tag')
+
 
 def wipe_database():
   logger.debug('Wiping all data from the DB.')
@@ -910,7 +913,7 @@ def populate_database(minimal=False, with_storage=False):
     model.repositoryactioncount.update_repository_score(to_count)
 
 
-WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2']
+WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2', 'Tag', 'ManifestChild']
 
 def find_models_missing_data():
   # As a sanity check we are going to make sure that all db tables have some data, unless explicitly