From c0653ef2adbec86d38dbb10b311903e28f6bc81e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 29 Oct 2018 15:37:33 -0400 Subject: [PATCH] Add Tag, TagKind and ManifestChild tables in prep for new data model --- data/database.py | 55 ++++++++++- ...dd_tag_tagkind_and_manifestchild_tables.py | 97 +++++++++++++++++++ initdb.py | 7 +- 3 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 data/migrations/versions/10f45ee2310b_add_tag_tagkind_and_manifestchild_tables.py diff --git a/data/database.py b/data/database.py index 506560469..e25c28cac 100644 --- a/data/database.py +++ b/data/database.py @@ -1385,6 +1385,58 @@ class Manifest(BaseModel): ) +class TagKind(BaseModel): + """ TagKind describes the various kinds of tags that can be found in the registry. + """ + name = CharField(index=True, unique=True) + + +class Tag(BaseModel): + """ Tag represents a user-facing alias for referencing a Manifest or as an alias to another tag. + """ + name = CharField() + repository = ForeignKeyField(Repository) + manifest = ForeignKeyField(Manifest, null=True) + lifetime_start_ms = BigIntegerField(default=get_epoch_timestamp_ms) + lifetime_end_ms = BigIntegerField(null=True, index=True) + hidden = BooleanField(default=False) + reverted = BooleanField(default=False) + tag_kind = EnumField(TagKind) + linked_tag = ForeignKeyField('self', null=True, backref='tag_parents') + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('repository', 'name'), False), + (('repository', 'name', 'hidden'), False), + (('repository', 'name', 'tag_kind'), False), + + # This unique index prevents deadlocks when concurrently moving and deleting tags + (('repository', 'name', 'lifetime_end_ms'), True), + ) + + +class ManifestChild(BaseModel): + """ ManifestChild represents a relationship between a manifest and its child manifest(s). + Multiple manifests can share the same children. Note that since Manifests are stored + per-repository, the repository here is a bit redundant, but we do so to make cleanup easier. + """ + repository = ForeignKeyField(Repository) + manifest = ForeignKeyField(Manifest) + child_manifest = ForeignKeyField(Manifest) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('repository', 'manifest'), False), + (('repository', 'child_manifest'), False), + (('repository', 'manifest', 'child_manifest'), False), + (('manifest', 'child_manifest'), True), + ) + + class ManifestLabel(BaseModel): """ ManifestLabel represents a label applied to a Manifest, within a repository. Note that since Manifests are stored per-repository, the repository here is @@ -1470,7 +1522,8 @@ class TagManifestLabelMap(BaseModel): appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, ApprBlobPlacement]) -v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage]) +v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage, TagKind, + ManifestChild, Tag]) transition_classes = set([TagManifestToManifest, TagManifestLabelMap]) is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel diff --git a/data/migrations/versions/10f45ee2310b_add_tag_tagkind_and_manifestchild_tables.py b/data/migrations/versions/10f45ee2310b_add_tag_tagkind_and_manifestchild_tables.py new file mode 100644 index 000000000..a29fd5fdc --- /dev/null +++ b/data/migrations/versions/10f45ee2310b_add_tag_tagkind_and_manifestchild_tables.py @@ -0,0 +1,97 @@ +"""Add Tag, TagKind and ManifestChild tables + +Revision ID: 10f45ee2310b +Revises: 13411de1c0ff +Create Date: 2018-10-29 15:22:53.552216 + +""" + +# revision identifiers, used by Alembic. +revision = '10f45ee2310b' +down_revision = '13411de1c0ff' + +from alembic import op +import sqlalchemy as sa +from util.migrate import UTF8CharField + +def upgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tagkind', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_tagkind')) + ) + op.create_index('tagkind_name', 'tagkind', ['name'], unique=True) + op.create_table('manifestchild', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('manifest_id', sa.Integer(), nullable=False), + sa.Column('child_manifest_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['child_manifest_id'], ['manifest.id'], name=op.f('fk_manifestchild_child_manifest_id_manifest')), + sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestchild_manifest_id_manifest')), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_manifestchild_repository_id_repository')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestchild')) + ) + op.create_index('manifestchild_child_manifest_id', 'manifestchild', ['child_manifest_id'], unique=False) + op.create_index('manifestchild_manifest_id', 'manifestchild', ['manifest_id'], unique=False) + op.create_index('manifestchild_manifest_id_child_manifest_id', 'manifestchild', ['manifest_id', 'child_manifest_id'], unique=True) + op.create_index('manifestchild_repository_id', 'manifestchild', ['repository_id'], unique=False) + op.create_index('manifestchild_repository_id_child_manifest_id', 'manifestchild', ['repository_id', 'child_manifest_id'], unique=False) + op.create_index('manifestchild_repository_id_manifest_id', 'manifestchild', ['repository_id', 'manifest_id'], unique=False) + op.create_index('manifestchild_repository_id_manifest_id_child_manifest_id', 'manifestchild', ['repository_id', 'manifest_id', 'child_manifest_id'], unique=False) + op.create_table('tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', UTF8CharField(length=255), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('manifest_id', sa.Integer(), nullable=True), + sa.Column('lifetime_start_ms', sa.BigInteger(), nullable=False), + sa.Column('lifetime_end_ms', sa.BigInteger(), nullable=True), + sa.Column('hidden', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()), + sa.Column('reverted', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()), + sa.Column('tag_kind_id', sa.Integer(), nullable=False), + sa.Column('linked_tag_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['linked_tag_id'], ['tag.id'], name=op.f('fk_tag_linked_tag_id_tag')), + sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_tag_manifest_id_manifest')), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tag_repository_id_repository')), + sa.ForeignKeyConstraint(['tag_kind_id'], ['tagkind.id'], name=op.f('fk_tag_tag_kind_id_tagkind')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_tag')) + ) + op.create_index('tag_lifetime_end_ms', 'tag', ['lifetime_end_ms'], unique=False) + op.create_index('tag_linked_tag_id', 'tag', ['linked_tag_id'], unique=False) + op.create_index('tag_manifest_id', 'tag', ['manifest_id'], unique=False) + op.create_index('tag_repository_id', 'tag', ['repository_id'], unique=False) + op.create_index('tag_repository_id_name', 'tag', ['repository_id', 'name'], unique=False) + op.create_index('tag_repository_id_name_hidden', 'tag', ['repository_id', 'name', 'hidden'], unique=False) + op.create_index('tag_repository_id_name_lifetime_end_ms', 'tag', ['repository_id', 'name', 'lifetime_end_ms'], unique=True) + op.create_index('tag_repository_id_name_tag_kind_id', 'tag', ['repository_id', 'name', 'tag_kind_id'], unique=False) + op.create_index('tag_tag_kind_id', 'tag', ['tag_kind_id'], unique=False) + # ### end Alembic commands ### + + op.bulk_insert(tables.tagkind, + [ + {'name': 'tag'}, + ]) + + # ### population of test data ### # + tester.populate_table('tag', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('tag_kind_id', tester.TestDataType.Foreign('tagkind')), + ('name', tester.TestDataType.String), + ('manifest_id', tester.TestDataType.Foreign('manifest')), + ('lifetime_start_ms', tester.TestDataType.BigInteger), + ]) + + tester.populate_table('manifestchild', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('manifest_id', tester.TestDataType.Foreign('manifest')), + ('child_manifest_id', tester.TestDataType.Foreign('manifest')), + ]) + # ### end population of test data ### # + + +def downgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tag') + op.drop_table('manifestchild') + op.drop_table('tagkind') + # ### end Alembic commands ### diff --git a/initdb.py b/initdb.py index 67c9acbdf..0b4d9994c 100644 --- a/initdb.py +++ b/initdb.py @@ -20,7 +20,8 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode, ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind, RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes, - ApprTagKind, ApprBlobPlacementLocation, Repository) + ApprTagKind, ApprBlobPlacementLocation, Repository, Tag, TagKind, + ManifestChild) from data import model from data.queue import WorkQueue from data.registry_model import registry_model @@ -450,6 +451,8 @@ def initialize_database(): DisableReason.create(name='successive_build_failures') DisableReason.create(name='successive_build_internal_errors') + TagKind.create(name='tag') + def wipe_database(): logger.debug('Wiping all data from the DB.') @@ -910,7 +913,7 @@ def populate_database(minimal=False, with_storage=False): model.repositoryactioncount.update_repository_score(to_count) -WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2'] +WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2', 'Tag', 'ManifestChild'] def find_models_missing_data(): # As a sanity check we are going to make sure that all db tables have some data, unless explicitly