Phase 3 of Appr migration
Deletes the old models and their code
This commit is contained in:
Joseph Schorr
parent
223077ef53
commit
487edf0ba1
9 changed files with 218 additions and 167 deletions
|
|
@ -1,7 +1,5 @@
|
|||
from collections import namedtuple
|
||||
|
||||
from data.database import (Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
|
||||
ManifestListManifest, Manifest, BlobPlacement, Channel)
|
||||
from data.database import (ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
|
||||
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
|
||||
ApprBlobPlacement, ApprChannel)
|
||||
|
|
@ -11,10 +9,6 @@ ModelsRef = namedtuple('ModelsRef', ['Tag', 'TagKind', 'BlobPlacementLocation',
|
|||
'BlobPlacement', 'Channel', 'manifestlistmanifest_set_name',
|
||||
'tag_set_prefetch_name'])
|
||||
|
||||
OLD_MODELS = ModelsRef(Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
|
||||
ManifestListManifest, Manifest, BlobPlacement, Channel,
|
||||
'manifestlistmanifest_set', 'tag_set_prefetch')
|
||||
|
||||
NEW_MODELS = ModelsRef(ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
|
||||
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
|
||||
ApprBlobPlacement, ApprChannel, 'apprmanifestlistmanifest_set',
|
||||
|
|
|
|||
127
data/database.py
127
data/database.py
|
|
@ -471,7 +471,7 @@ class User(BaseModel):
|
|||
TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
|
||||
RepositoryNotification, OAuthAuthorizationCode,
|
||||
RepositoryActionCount, TagManifestLabel,
|
||||
TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes | appr_classes
|
||||
TeamSync, RepositorySearchScore, DeletedNamespace} | appr_classes
|
||||
delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
|
||||
|
||||
|
||||
|
|
@ -619,7 +619,7 @@ class Repository(BaseModel):
|
|||
# are cleaned up directly
|
||||
skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
|
||||
Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage,
|
||||
RepositorySearchScore} | cnr_classes | appr_classes
|
||||
RepositorySearchScore} | appr_classes
|
||||
|
||||
delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
|
||||
|
||||
|
|
@ -1231,17 +1231,6 @@ class TagManifestLabel(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class Blob(BaseModel):
|
||||
""" Blob represents a content-addressable object stored outside of the database.
|
||||
This is deprecated in favor of ApprBlob.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
media_type = EnumField(MediaType)
|
||||
size = BigIntegerField()
|
||||
uncompressed_size = BigIntegerField(null=True)
|
||||
|
||||
|
||||
|
||||
class ApprBlob(BaseModel):
|
||||
""" ApprBlob represents a content-addressable object stored outside of the database.
|
||||
"""
|
||||
|
|
@ -1251,34 +1240,12 @@ class ApprBlob(BaseModel):
|
|||
uncompressed_size = BigIntegerField(null=True)
|
||||
|
||||
|
||||
class BlobPlacementLocation(BaseModel):
|
||||
""" BlobPlacementLocation is an enumeration of the possible storage locations for Blobs.
|
||||
This is deprecated in favor of ApprBlobPlacementLocation.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class ApprBlobPlacementLocation(BaseModel):
|
||||
""" ApprBlobPlacementLocation is an enumeration of the possible storage locations for ApprBlobs.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class BlobPlacement(BaseModel):
|
||||
""" BlobPlacement represents the location of a Blob.
|
||||
This is deprecated in favor of ApprBlobPlacement.
|
||||
"""
|
||||
blob = ForeignKeyField(Blob)
|
||||
location = EnumField(BlobPlacementLocation)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('blob', 'location'), True),
|
||||
)
|
||||
|
||||
|
||||
class ApprBlobPlacement(BaseModel):
|
||||
""" ApprBlobPlacement represents the location of a Blob.
|
||||
"""
|
||||
|
|
@ -1293,15 +1260,6 @@ class ApprBlobPlacement(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class Manifest(BaseModel):
|
||||
""" Manifest represents the metadata and collection of blobs that comprise a container image.
|
||||
This is deprecated in favor of ApprManifest.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
media_type = EnumField(MediaType)
|
||||
manifest_json = JSONField()
|
||||
|
||||
|
||||
class ApprManifest(BaseModel):
|
||||
""" ApprManifest represents the metadata and collection of blobs that comprise an Appr image.
|
||||
"""
|
||||
|
|
@ -1310,21 +1268,6 @@ class ApprManifest(BaseModel):
|
|||
manifest_json = JSONField()
|
||||
|
||||
|
||||
class ManifestBlob(BaseModel):
|
||||
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs.
|
||||
This is deprecated in favor of ApprManifestBlob.
|
||||
"""
|
||||
manifest = ForeignKeyField(Manifest, index=True)
|
||||
blob = ForeignKeyField(Blob, index=True)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('manifest', 'blob'), True),
|
||||
)
|
||||
|
||||
|
||||
class ApprManifestBlob(BaseModel):
|
||||
""" ApprManifestBlob is a many-to-many relation table linking ApprManifests and ApprBlobs.
|
||||
"""
|
||||
|
|
@ -1339,16 +1282,6 @@ class ApprManifestBlob(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ManifestList(BaseModel):
|
||||
""" ManifestList represents all of the various manifests that compose a Tag.
|
||||
This is deprecated in favor of ApprManifestList.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
manifest_list_json = JSONField()
|
||||
schema_version = CharField()
|
||||
media_type = EnumField(MediaType)
|
||||
|
||||
|
||||
class ApprManifestList(BaseModel):
|
||||
""" ApprManifestList represents all of the various Appr manifests that compose an ApprTag.
|
||||
"""
|
||||
|
|
@ -1358,45 +1291,12 @@ class ApprManifestList(BaseModel):
|
|||
media_type = EnumField(MediaType)
|
||||
|
||||
|
||||
class TagKind(BaseModel):
|
||||
""" TagKind is a enumtable to reference tag kinds.
|
||||
This model is deprecated in favor of ApprTagKind.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class ApprTagKind(BaseModel):
|
||||
""" ApprTagKind is a enumtable to reference tag kinds.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class Tag(BaseModel):
|
||||
""" Tag represents a user-facing alias for referencing a ManifestList.
|
||||
This model is deprecated in favor of ApprTag.
|
||||
"""
|
||||
name = CharField()
|
||||
repository = ForeignKeyField(Repository)
|
||||
manifest_list = ForeignKeyField(ManifestList, null=True)
|
||||
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
|
||||
lifetime_end = BigIntegerField(null=True, index=True)
|
||||
hidden = BooleanField(default=False)
|
||||
reverted = BooleanField(default=False)
|
||||
protected = BooleanField(default=False)
|
||||
tag_kind = EnumField(TagKind)
|
||||
linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('repository', 'name'), False),
|
||||
(('repository', 'name', 'hidden'), False),
|
||||
# This unique index prevents deadlocks when concurrently moving and deleting tags
|
||||
(('repository', 'name', 'lifetime_end'), True),
|
||||
)
|
||||
|
||||
|
||||
class ApprTag(BaseModel):
|
||||
""" ApprTag represents a user-facing alias for referencing an ApprManifestList.
|
||||
"""
|
||||
|
|
@ -1421,30 +1321,9 @@ class ApprTag(BaseModel):
|
|||
(('repository', 'name', 'lifetime_end'), True),
|
||||
)
|
||||
|
||||
Channel = Tag.alias()
|
||||
ApprChannel = ApprTag.alias()
|
||||
|
||||
|
||||
class ManifestListManifest(BaseModel):
|
||||
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests.
|
||||
This model is deprecated in favor of ApprManifestListManifest.
|
||||
"""
|
||||
manifest_list = ForeignKeyField(ManifestList, index=True)
|
||||
manifest = ForeignKeyField(Manifest, index=True)
|
||||
operating_system = CharField(null=True)
|
||||
architecture = CharField(null=True)
|
||||
platform_json = JSONField(null=True)
|
||||
media_type = EnumField(MediaType)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('manifest_list', 'operating_system', 'architecture', 'media_type'), False),
|
||||
(('manifest_list', 'media_type'), False),
|
||||
)
|
||||
|
||||
|
||||
class ApprManifestListManifest(BaseModel):
|
||||
""" ApprManifestListManifest is a many-to-many relation table linking ApprManifestLists and
|
||||
ApprManifests.
|
||||
|
|
@ -1484,8 +1363,6 @@ class AppSpecificAuthToken(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
|
||||
ManifestListManifest, Manifest, BlobPlacement])
|
||||
appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
|
||||
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
|
||||
ApprBlobPlacement])
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ down_revision = 'e2894a3a3c19'
|
|||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import mysql
|
||||
from sqlalchemy.sql import table, column
|
||||
from util.migrate import UTF8LongText, UTF8CharField
|
||||
|
||||
|
||||
|
|
@ -274,8 +275,13 @@ def upgrade(tables, tester):
|
|||
)
|
||||
op.create_index('manifestlayerscan_layer_id', 'manifestlayerscan', ['layer_id'], unique=True)
|
||||
|
||||
blobplacementlocation_table = table('blobplacementlocation',
|
||||
column('id', sa.Integer()),
|
||||
column('name', sa.String()),
|
||||
)
|
||||
|
||||
op.bulk_insert(
|
||||
tables.blobplacementlocation,
|
||||
blobplacementlocation_table,
|
||||
[
|
||||
{'name': 'local_eu'},
|
||||
{'name': 'local_us'},
|
||||
|
|
@ -297,8 +303,13 @@ def upgrade(tables, tester):
|
|||
],
|
||||
)
|
||||
|
||||
tagkind_table = table('tagkind',
|
||||
column('id', sa.Integer()),
|
||||
column('name', sa.String()),
|
||||
)
|
||||
|
||||
op.bulk_insert(
|
||||
tables.tagkind,
|
||||
tagkind_table,
|
||||
[
|
||||
{'id': 1, 'name': 'tag'},
|
||||
{'id': 2, 'name': 'release'},
|
||||
|
|
@ -306,7 +317,6 @@ def upgrade(tables, tester):
|
|||
]
|
||||
)
|
||||
|
||||
|
||||
def downgrade(tables, tester):
|
||||
op.drop_table('manifestlayerscan')
|
||||
op.drop_table('manifestlayerdockerv1')
|
||||
|
|
|
|||
189
data/migrations/versions/d17c695859ea_delete_old_appr_tables.py
Normal file
189
data/migrations/versions/d17c695859ea_delete_old_appr_tables.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
"""Delete old Appr tables
|
||||
|
||||
Revision ID: d17c695859ea
|
||||
Revises: 5d463ea1e8a8
|
||||
Create Date: 2018-07-16 15:21:11.593040
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'd17c695859ea'
|
||||
down_revision = '5d463ea1e8a8'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.sql import table, column
|
||||
from util.migrate import UTF8LongText, UTF8CharField
|
||||
|
||||
def upgrade(tables, tester):
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('tag')
|
||||
op.drop_table('manifestlistmanifest')
|
||||
op.drop_table('manifestlist')
|
||||
op.drop_table('manifestblob')
|
||||
op.drop_table('manifest')
|
||||
op.drop_table('blobplacement')
|
||||
op.drop_table('blob')
|
||||
op.drop_table('blobplacementlocation')
|
||||
op.drop_table('tagkind')
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade(tables, tester):
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
'tagkind',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_tagkind'))
|
||||
)
|
||||
op.create_index('tagkind_name', 'tagkind', ['name'], unique=True)
|
||||
|
||||
op.create_table(
|
||||
'blobplacementlocation',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacementlocation'))
|
||||
)
|
||||
op.create_index('blobplacementlocation_name', 'blobplacementlocation', ['name'], unique=True)
|
||||
|
||||
op.create_table(
|
||||
'blob',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('digest', sa.String(length=255), nullable=False),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.Column('size', sa.BigInteger(), nullable=False),
|
||||
sa.Column('uncompressed_size', sa.BigInteger(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_blob_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_blob'))
|
||||
)
|
||||
op.create_index('blob_digest', 'blob', ['digest'], unique=True)
|
||||
op.create_index('blob_media_type_id', 'blob', ['media_type_id'], unique=False)
|
||||
|
||||
op.create_table(
|
||||
'manifest',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('digest', sa.String(length=255), nullable=False),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_json', UTF8LongText, nullable=False),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifest_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifest'))
|
||||
)
|
||||
op.create_index('manifest_digest', 'manifest', ['digest'], unique=True)
|
||||
op.create_index('manifest_media_type_id', 'manifest', ['media_type_id'], unique=False)
|
||||
|
||||
op.create_table(
|
||||
'manifestlist',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('digest', sa.String(length=255), nullable=False),
|
||||
sa.Column('manifest_list_json', UTF8LongText, nullable=False),
|
||||
sa.Column('schema_version', UTF8CharField(length=255), nullable=False),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlist_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlist'))
|
||||
)
|
||||
op.create_index('manifestlist_digest', 'manifestlist', ['digest'], unique=True)
|
||||
op.create_index('manifestlist_media_type_id', 'manifestlist', ['media_type_id'], unique=False)
|
||||
|
||||
op.create_table(
|
||||
'blobplacement',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('blob_id', sa.Integer(), nullable=False),
|
||||
sa.Column('location_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_blobplacement_blob_id_blob')),
|
||||
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobplacement_location_id_blobplacementlocation')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacement'))
|
||||
)
|
||||
op.create_index('blobplacement_blob_id', 'blobplacement', ['blob_id'], unique=False)
|
||||
op.create_index('blobplacement_blob_id_location_id', 'blobplacement', ['blob_id', 'location_id'], unique=True)
|
||||
op.create_index('blobplacement_location_id', 'blobplacement', ['location_id'], unique=False)
|
||||
|
||||
op.create_table(
|
||||
'manifestblob',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_id', sa.Integer(), nullable=False),
|
||||
sa.Column('blob_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_manifestblob_blob_id_blob')),
|
||||
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestblob_manifest_id_manifest')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestblob'))
|
||||
)
|
||||
op.create_index('manifestblob_blob_id', 'manifestblob', ['blob_id'], unique=False)
|
||||
op.create_index('manifestblob_manifest_id', 'manifestblob', ['manifest_id'], unique=False)
|
||||
op.create_index('manifestblob_manifest_id_blob_id', 'manifestblob', ['manifest_id', 'blob_id'], unique=True)
|
||||
|
||||
op.create_table(
|
||||
'manifestlistmanifest',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_list_id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_id', sa.Integer(), nullable=False),
|
||||
sa.Column('operating_system', UTF8CharField(length=255), nullable=True),
|
||||
sa.Column('architecture', UTF8CharField(length=255), nullable=True),
|
||||
sa.Column('platform_json', UTF8LongText, nullable=True),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestlistmanifest_manifest_id_manifest')),
|
||||
sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_manifestlistmanifest_manifest_list_id_manifestlist')),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlistmanifest_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlistmanifest'))
|
||||
)
|
||||
op.create_index('manifestlistmanifest_manifest_id', 'manifestlistmanifest', ['manifest_id'], unique=False)
|
||||
op.create_index('manifestlistmanifest_manifest_list_id', 'manifestlistmanifest', ['manifest_list_id'], unique=False)
|
||||
op.create_index('manifestlistmanifest_manifest_listid_os_arch_mtid', 'manifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False)
|
||||
op.create_index('manifestlistmanifest_manifest_listid_mtid', 'manifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False)
|
||||
op.create_index('manifestlistmanifest_media_type_id', 'manifestlistmanifest', ['media_type_id'], unique=False)
|
||||
|
||||
op.create_table(
|
||||
'tag',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', UTF8CharField(length=190), nullable=False),
|
||||
sa.Column('repository_id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_list_id', sa.Integer(), nullable=True),
|
||||
sa.Column('lifetime_start', sa.BigInteger(), nullable=False),
|
||||
sa.Column('lifetime_end', sa.BigInteger(), nullable=True),
|
||||
sa.Column('hidden', sa.Boolean(), nullable=False),
|
||||
sa.Column('reverted', sa.Boolean(), nullable=False),
|
||||
sa.Column('protected', sa.Boolean(), nullable=False),
|
||||
sa.Column('tag_kind_id', sa.Integer(), nullable=False),
|
||||
sa.Column('linked_tag_id', sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['linked_tag_id'], ['tag.id'], name=op.f('fk_tag_linked_tag_id_tag')),
|
||||
sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_tag_manifest_list_id_manifestlist')),
|
||||
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tag_repository_id_repository')),
|
||||
sa.ForeignKeyConstraint(['tag_kind_id'], ['tagkind.id'], name=op.f('fk_tag_tag_kind_id_tagkind')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
|
||||
)
|
||||
op.create_index('tag_lifetime_end', 'tag', ['lifetime_end'], unique=False)
|
||||
op.create_index('tag_linked_tag_id', 'tag', ['linked_tag_id'], unique=False)
|
||||
op.create_index('tag_manifest_list_id', 'tag', ['manifest_list_id'], unique=False)
|
||||
op.create_index('tag_repository_id', 'tag', ['repository_id'], unique=False)
|
||||
op.create_index('tag_repository_id_name_hidden', 'tag', ['repository_id', 'name', 'hidden'], unique=False)
|
||||
op.create_index('tag_repository_id_name_lifetime_end', 'tag', ['repository_id', 'name', 'lifetime_end'], unique=True)
|
||||
op.create_index('tag_repository_id_name', 'tag', ['repository_id', 'name'], unique=False)
|
||||
op.create_index('tag_tag_kind_id', 'tag', ['tag_kind_id'], unique=False)
|
||||
|
||||
# ### end Alembic commands ###
|
||||
|
||||
blobplacementlocation_table = table('blobplacementlocation',
|
||||
column('id', sa.Integer()),
|
||||
column('name', sa.String()),
|
||||
)
|
||||
|
||||
op.bulk_insert(
|
||||
blobplacementlocation_table,
|
||||
[
|
||||
{'name': 'local_eu'},
|
||||
{'name': 'local_us'},
|
||||
],
|
||||
)
|
||||
|
||||
tagkind_table = table('tagkind',
|
||||
column('id', sa.Integer()),
|
||||
column('name', sa.String()),
|
||||
)
|
||||
|
||||
op.bulk_insert(
|
||||
tagkind_table,
|
||||
[
|
||||
{'id': 1, 'name': 'tag'},
|
||||
{'id': 2, 'name': 'release'},
|
||||
{'id': 3, 'name': 'channel'},
|
||||
]
|
||||
)
|
||||
|
|
@ -9,11 +9,11 @@ from cachetools import ttl_cache
|
|||
|
||||
from data.model import (
|
||||
config, DataModelException, tag, db_transaction, storage, permission, _basequery)
|
||||
from data.database import (
|
||||
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag,
|
||||
from data.database import (
|
||||
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
|
||||
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
|
||||
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
|
||||
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind)
|
||||
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind, ApprTag)
|
||||
from data.text import prefix_search
|
||||
from util.itertoolrecipes import take
|
||||
|
||||
|
|
@ -90,8 +90,6 @@ def purge_repository(namespace_name, repository_name):
|
|||
# Delete the repository of all Appr-referenced entries.
|
||||
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
|
||||
# and MySQL is... particular... about such relationships when deleting.
|
||||
Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute()
|
||||
Tag.delete().where(Tag.repository == repo).execute()
|
||||
ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute()
|
||||
ApprTag.delete().where(ApprTag.repository == repo).execute()
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
|
|||
DataModelException, _basequery)
|
||||
from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
|
||||
ImageStorageTransformation, ImageStorageSignature,
|
||||
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob,
|
||||
ApprBlob, ensure_under_transaction)
|
||||
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, ApprBlob,
|
||||
ensure_under_transaction)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -105,21 +105,15 @@ def garbage_collect_storage(storage_id_whitelist):
|
|||
logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced',
|
||||
is_referenced_checksums)
|
||||
|
||||
# Check the ApprBlob tables as well.
|
||||
query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums))
|
||||
blob_referenced_checksums = set([blob.digest for blob in query])
|
||||
if blob_referenced_checksums:
|
||||
logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced',
|
||||
blob_referenced_checksums)
|
||||
|
||||
# Check the ApprBlob table as well.
|
||||
query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums))
|
||||
appr_blob_referenced_checksums = set([blob.digest for blob in query])
|
||||
if appr_blob_referenced_checksums:
|
||||
logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced',
|
||||
appr_blob_referenced_checksums)
|
||||
|
||||
unreferenced_checksums = (content_checksums - blob_referenced_checksums -
|
||||
appr_blob_referenced_checksums - is_referenced_checksums)
|
||||
unreferenced_checksums = (content_checksums - appr_blob_referenced_checksums -
|
||||
is_referenced_checksums)
|
||||
|
||||
# Return all placements for all image storages found not at a CAS path or with a content
|
||||
# checksum that is referenced.
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from playhouse.test_utils import assert_query_count
|
|||
|
||||
from data import model, database
|
||||
from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
|
||||
Blob, ApprBlob)
|
||||
ApprBlob)
|
||||
from test.fixtures import *
|
||||
|
||||
|
||||
|
|
@ -187,9 +187,6 @@ def assert_gc_integrity(expect_storage_removed=True):
|
|||
if storage_row.cas_path:
|
||||
storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
|
||||
|
||||
for blob_row in Blob.select():
|
||||
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
|
||||
|
||||
for blob_row in ApprBlob.select():
|
||||
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
|
||||
|
||||
|
|
@ -592,7 +589,6 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d
|
|||
media_type = database.MediaType.get(name='text/plain')
|
||||
|
||||
is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
|
||||
database.Blob.create(digest=digest, size=0, media_type=media_type)
|
||||
database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
|
||||
|
||||
location = database.ImageStorageLocation.get(name=preferred)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import data.model
|
|||
from app import storage, authentication
|
||||
from data import appr_model
|
||||
from data.database import Repository, MediaType, db_transaction
|
||||
from data.appr_model.models import OLD_MODELS, NEW_MODELS
|
||||
from data.appr_model.models import NEW_MODELS
|
||||
from endpoints.appr.models_interface import (
|
||||
ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface,
|
||||
BlobDescriptor, ChannelView, ChannelReleasesView)
|
||||
|
|
|
|||
15
initdb.py
15
initdb.py
|
|
@ -14,14 +14,14 @@ from uuid import UUID, uuid4
|
|||
from threading import Event
|
||||
|
||||
from email.utils import formatdate
|
||||
from data.database import (db, all_models, cnr_classes, Role, TeamRole, Visibility, LoginService,
|
||||
from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService,
|
||||
BuildTriggerService, AccessTokenKind, LogEntryKind, ImageStorageLocation,
|
||||
ImageStorageTransformation, ImageStorageSignatureKind,
|
||||
ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind,
|
||||
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
|
||||
ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
|
||||
RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason,
|
||||
DeletedNamespace, appr_classes, ApprTagKind, ApprBlobPlacementLocation)
|
||||
RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes,
|
||||
ApprTagKind, ApprBlobPlacementLocation)
|
||||
from data import model
|
||||
from data.queue import WorkQueue
|
||||
from app import app, storage as store, tf
|
||||
|
|
@ -361,9 +361,6 @@ def initialize_database():
|
|||
ImageStorageLocation.create(name='local_eu')
|
||||
ImageStorageLocation.create(name='local_us')
|
||||
|
||||
BlobPlacementLocation.create(name='local_eu')
|
||||
BlobPlacementLocation.create(name='local_us')
|
||||
|
||||
ApprBlobPlacementLocation.create(name='local_eu')
|
||||
ApprBlobPlacementLocation.create(name='local_us')
|
||||
|
||||
|
|
@ -434,10 +431,6 @@ def initialize_database():
|
|||
RepositoryKind.create(name='image')
|
||||
RepositoryKind.create(name='application')
|
||||
|
||||
TagKind.create(name='tag')
|
||||
TagKind.create(name='release')
|
||||
TagKind.create(name='channel')
|
||||
|
||||
ApprTagKind.create(name='tag')
|
||||
ApprTagKind.create(name='release')
|
||||
ApprTagKind.create(name='channel')
|
||||
|
|
@ -913,7 +906,7 @@ def find_models_missing_data():
|
|||
# whitelisted.
|
||||
models_missing_data = set()
|
||||
for one_model in all_models:
|
||||
if one_model in cnr_classes or one_model in appr_classes:
|
||||
if one_model in appr_classes:
|
||||
continue
|
||||
|
||||
try:
|
||||
|
|
|
|||
Reference in a new issue