diff --git a/data/appr_model/models.py b/data/appr_model/models.py index 87f41eb8f..9903e97b5 100644 --- a/data/appr_model/models.py +++ b/data/appr_model/models.py @@ -1,7 +1,5 @@ from collections import namedtuple -from data.database import (Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob, - ManifestListManifest, Manifest, BlobPlacement, Channel) from data.database import (ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, ApprBlobPlacement, ApprChannel) @@ -11,10 +9,6 @@ ModelsRef = namedtuple('ModelsRef', ['Tag', 'TagKind', 'BlobPlacementLocation', 'BlobPlacement', 'Channel', 'manifestlistmanifest_set_name', 'tag_set_prefetch_name']) -OLD_MODELS = ModelsRef(Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob, - ManifestListManifest, Manifest, BlobPlacement, Channel, - 'manifestlistmanifest_set', 'tag_set_prefetch') - NEW_MODELS = ModelsRef(ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, ApprBlobPlacement, ApprChannel, 'apprmanifestlistmanifest_set', diff --git a/data/database.py b/data/database.py index ef88615b6..312e66358 100644 --- a/data/database.py +++ b/data/database.py @@ -471,7 +471,7 @@ class User(BaseModel): TagManifest, AccessToken, OAuthAccessToken, BlobUpload, RepositoryNotification, OAuthAuthorizationCode, RepositoryActionCount, TagManifestLabel, - TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes | appr_classes + TeamSync, RepositorySearchScore, DeletedNamespace} | appr_classes delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes) @@ -619,7 +619,7 @@ class Repository(BaseModel): # are cleaned up directly skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload, Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage, - RepositorySearchScore} | cnr_classes | appr_classes + RepositorySearchScore} | appr_classes delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes) @@ -1231,17 +1231,6 @@ class TagManifestLabel(BaseModel): ) -class Blob(BaseModel): - """ Blob represents a content-addressable object stored outside of the database. - This is deprecated in favor of ApprBlob. - """ - digest = CharField(index=True, unique=True) - media_type = EnumField(MediaType) - size = BigIntegerField() - uncompressed_size = BigIntegerField(null=True) - - - class ApprBlob(BaseModel): """ ApprBlob represents a content-addressable object stored outside of the database. """ @@ -1251,34 +1240,12 @@ class ApprBlob(BaseModel): uncompressed_size = BigIntegerField(null=True) -class BlobPlacementLocation(BaseModel): - """ BlobPlacementLocation is an enumeration of the possible storage locations for Blobs. - This is deprecated in favor of ApprBlobPlacementLocation. - """ - name = CharField(index=True, unique=True) - - class ApprBlobPlacementLocation(BaseModel): """ ApprBlobPlacementLocation is an enumeration of the possible storage locations for ApprBlobs. """ name = CharField(index=True, unique=True) -class BlobPlacement(BaseModel): - """ BlobPlacement represents the location of a Blob. - This is deprecated in favor of ApprBlobPlacement. - """ - blob = ForeignKeyField(Blob) - location = EnumField(BlobPlacementLocation) - - class Meta: - database = db - read_slaves = (read_slave,) - indexes = ( - (('blob', 'location'), True), - ) - - class ApprBlobPlacement(BaseModel): """ ApprBlobPlacement represents the location of a Blob. """ @@ -1293,15 +1260,6 @@ class ApprBlobPlacement(BaseModel): ) -class Manifest(BaseModel): - """ Manifest represents the metadata and collection of blobs that comprise a container image. - This is deprecated in favor of ApprManifest. - """ - digest = CharField(index=True, unique=True) - media_type = EnumField(MediaType) - manifest_json = JSONField() - - class ApprManifest(BaseModel): """ ApprManifest represents the metadata and collection of blobs that comprise an Appr image. """ @@ -1310,21 +1268,6 @@ class ApprManifest(BaseModel): manifest_json = JSONField() -class ManifestBlob(BaseModel): - """ ManifestBlob is a many-to-many relation table linking Manifests and Blobs. - This is deprecated in favor of ApprManifestBlob. - """ - manifest = ForeignKeyField(Manifest, index=True) - blob = ForeignKeyField(Blob, index=True) - - class Meta: - database = db - read_slaves = (read_slave,) - indexes = ( - (('manifest', 'blob'), True), - ) - - class ApprManifestBlob(BaseModel): """ ApprManifestBlob is a many-to-many relation table linking ApprManifests and ApprBlobs. """ @@ -1339,16 +1282,6 @@ class ApprManifestBlob(BaseModel): ) -class ManifestList(BaseModel): - """ ManifestList represents all of the various manifests that compose a Tag. - This is deprecated in favor of ApprManifestList. - """ - digest = CharField(index=True, unique=True) - manifest_list_json = JSONField() - schema_version = CharField() - media_type = EnumField(MediaType) - - class ApprManifestList(BaseModel): """ ApprManifestList represents all of the various Appr manifests that compose an ApprTag. """ @@ -1358,45 +1291,12 @@ class ApprManifestList(BaseModel): media_type = EnumField(MediaType) -class TagKind(BaseModel): - """ TagKind is a enumtable to reference tag kinds. - This model is deprecated in favor of ApprTagKind. - """ - name = CharField(index=True, unique=True) - class ApprTagKind(BaseModel): """ ApprTagKind is a enumtable to reference tag kinds. """ name = CharField(index=True, unique=True) - -class Tag(BaseModel): - """ Tag represents a user-facing alias for referencing a ManifestList. - This model is deprecated in favor of ApprTag. - """ - name = CharField() - repository = ForeignKeyField(Repository) - manifest_list = ForeignKeyField(ManifestList, null=True) - lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms) - lifetime_end = BigIntegerField(null=True, index=True) - hidden = BooleanField(default=False) - reverted = BooleanField(default=False) - protected = BooleanField(default=False) - tag_kind = EnumField(TagKind) - linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents') - - class Meta: - database = db - read_slaves = (read_slave,) - indexes = ( - (('repository', 'name'), False), - (('repository', 'name', 'hidden'), False), - # This unique index prevents deadlocks when concurrently moving and deleting tags - (('repository', 'name', 'lifetime_end'), True), - ) - - class ApprTag(BaseModel): """ ApprTag represents a user-facing alias for referencing an ApprManifestList. """ @@ -1421,30 +1321,9 @@ class ApprTag(BaseModel): (('repository', 'name', 'lifetime_end'), True), ) -Channel = Tag.alias() ApprChannel = ApprTag.alias() -class ManifestListManifest(BaseModel): - """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. - This model is deprecated in favor of ApprManifestListManifest. - """ - manifest_list = ForeignKeyField(ManifestList, index=True) - manifest = ForeignKeyField(Manifest, index=True) - operating_system = CharField(null=True) - architecture = CharField(null=True) - platform_json = JSONField(null=True) - media_type = EnumField(MediaType) - - class Meta: - database = db - read_slaves = (read_slave,) - indexes = ( - (('manifest_list', 'operating_system', 'architecture', 'media_type'), False), - (('manifest_list', 'media_type'), False), - ) - - class ApprManifestListManifest(BaseModel): """ ApprManifestListManifest is a many-to-many relation table linking ApprManifestLists and ApprManifests. @@ -1484,8 +1363,6 @@ class AppSpecificAuthToken(BaseModel): ) -cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob, - ManifestListManifest, Manifest, BlobPlacement]) appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, ApprBlobPlacement]) diff --git a/data/migrations/versions/7a525c68eb13_add_oci_app_models.py b/data/migrations/versions/7a525c68eb13_add_oci_app_models.py index 1f97274d2..36c1f9ebd 100644 --- a/data/migrations/versions/7a525c68eb13_add_oci_app_models.py +++ b/data/migrations/versions/7a525c68eb13_add_oci_app_models.py @@ -13,6 +13,7 @@ down_revision = 'e2894a3a3c19' from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql +from sqlalchemy.sql import table, column from util.migrate import UTF8LongText, UTF8CharField @@ -274,8 +275,13 @@ def upgrade(tables, tester): ) op.create_index('manifestlayerscan_layer_id', 'manifestlayerscan', ['layer_id'], unique=True) + blobplacementlocation_table = table('blobplacementlocation', + column('id', sa.Integer()), + column('name', sa.String()), + ) + op.bulk_insert( - tables.blobplacementlocation, + blobplacementlocation_table, [ {'name': 'local_eu'}, {'name': 'local_us'}, @@ -297,8 +303,13 @@ def upgrade(tables, tester): ], ) + tagkind_table = table('tagkind', + column('id', sa.Integer()), + column('name', sa.String()), + ) + op.bulk_insert( - tables.tagkind, + tagkind_table, [ {'id': 1, 'name': 'tag'}, {'id': 2, 'name': 'release'}, @@ -306,7 +317,6 @@ def upgrade(tables, tester): ] ) - def downgrade(tables, tester): op.drop_table('manifestlayerscan') op.drop_table('manifestlayerdockerv1') diff --git a/data/migrations/versions/d17c695859ea_delete_old_appr_tables.py b/data/migrations/versions/d17c695859ea_delete_old_appr_tables.py new file mode 100644 index 000000000..777cfa112 --- /dev/null +++ b/data/migrations/versions/d17c695859ea_delete_old_appr_tables.py @@ -0,0 +1,189 @@ +"""Delete old Appr tables + +Revision ID: d17c695859ea +Revises: 5d463ea1e8a8 +Create Date: 2018-07-16 15:21:11.593040 + +""" + +# revision identifiers, used by Alembic. +revision = 'd17c695859ea' +down_revision = '5d463ea1e8a8' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import table, column +from util.migrate import UTF8LongText, UTF8CharField + +def upgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tag') + op.drop_table('manifestlistmanifest') + op.drop_table('manifestlist') + op.drop_table('manifestblob') + op.drop_table('manifest') + op.drop_table('blobplacement') + op.drop_table('blob') + op.drop_table('blobplacementlocation') + op.drop_table('tagkind') + # ### end Alembic commands ### + + +def downgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'tagkind', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_tagkind')) + ) + op.create_index('tagkind_name', 'tagkind', ['name'], unique=True) + + op.create_table( + 'blobplacementlocation', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacementlocation')) + ) + op.create_index('blobplacementlocation_name', 'blobplacementlocation', ['name'], unique=True) + + op.create_table( + 'blob', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('digest', sa.String(length=255), nullable=False), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.Column('size', sa.BigInteger(), nullable=False), + sa.Column('uncompressed_size', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_blob_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_blob')) + ) + op.create_index('blob_digest', 'blob', ['digest'], unique=True) + op.create_index('blob_media_type_id', 'blob', ['media_type_id'], unique=False) + + op.create_table( + 'manifest', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('digest', sa.String(length=255), nullable=False), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.Column('manifest_json', UTF8LongText, nullable=False), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifest_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_manifest')) + ) + op.create_index('manifest_digest', 'manifest', ['digest'], unique=True) + op.create_index('manifest_media_type_id', 'manifest', ['media_type_id'], unique=False) + + op.create_table( + 'manifestlist', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('digest', sa.String(length=255), nullable=False), + sa.Column('manifest_list_json', UTF8LongText, nullable=False), + sa.Column('schema_version', UTF8CharField(length=255), nullable=False), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlist_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlist')) + ) + op.create_index('manifestlist_digest', 'manifestlist', ['digest'], unique=True) + op.create_index('manifestlist_media_type_id', 'manifestlist', ['media_type_id'], unique=False) + + op.create_table( + 'blobplacement', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('blob_id', sa.Integer(), nullable=False), + sa.Column('location_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_blobplacement_blob_id_blob')), + sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobplacement_location_id_blobplacementlocation')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacement')) + ) + op.create_index('blobplacement_blob_id', 'blobplacement', ['blob_id'], unique=False) + op.create_index('blobplacement_blob_id_location_id', 'blobplacement', ['blob_id', 'location_id'], unique=True) + op.create_index('blobplacement_location_id', 'blobplacement', ['location_id'], unique=False) + + op.create_table( + 'manifestblob', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('manifest_id', sa.Integer(), nullable=False), + sa.Column('blob_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_manifestblob_blob_id_blob')), + sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestblob_manifest_id_manifest')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestblob')) + ) + op.create_index('manifestblob_blob_id', 'manifestblob', ['blob_id'], unique=False) + op.create_index('manifestblob_manifest_id', 'manifestblob', ['manifest_id'], unique=False) + op.create_index('manifestblob_manifest_id_blob_id', 'manifestblob', ['manifest_id', 'blob_id'], unique=True) + + op.create_table( + 'manifestlistmanifest', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('manifest_list_id', sa.Integer(), nullable=False), + sa.Column('manifest_id', sa.Integer(), nullable=False), + sa.Column('operating_system', UTF8CharField(length=255), nullable=True), + sa.Column('architecture', UTF8CharField(length=255), nullable=True), + sa.Column('platform_json', UTF8LongText, nullable=True), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestlistmanifest_manifest_id_manifest')), + sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_manifestlistmanifest_manifest_list_id_manifestlist')), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlistmanifest_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlistmanifest')) + ) + op.create_index('manifestlistmanifest_manifest_id', 'manifestlistmanifest', ['manifest_id'], unique=False) + op.create_index('manifestlistmanifest_manifest_list_id', 'manifestlistmanifest', ['manifest_list_id'], unique=False) + op.create_index('manifestlistmanifest_manifest_listid_os_arch_mtid', 'manifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False) + op.create_index('manifestlistmanifest_manifest_listid_mtid', 'manifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False) + op.create_index('manifestlistmanifest_media_type_id', 'manifestlistmanifest', ['media_type_id'], unique=False) + + op.create_table( + 'tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', UTF8CharField(length=190), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('manifest_list_id', sa.Integer(), nullable=True), + sa.Column('lifetime_start', sa.BigInteger(), nullable=False), + sa.Column('lifetime_end', sa.BigInteger(), nullable=True), + sa.Column('hidden', sa.Boolean(), nullable=False), + sa.Column('reverted', sa.Boolean(), nullable=False), + sa.Column('protected', sa.Boolean(), nullable=False), + sa.Column('tag_kind_id', sa.Integer(), nullable=False), + sa.Column('linked_tag_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['linked_tag_id'], ['tag.id'], name=op.f('fk_tag_linked_tag_id_tag')), + sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_tag_manifest_list_id_manifestlist')), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tag_repository_id_repository')), + sa.ForeignKeyConstraint(['tag_kind_id'], ['tagkind.id'], name=op.f('fk_tag_tag_kind_id_tagkind')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_tag')) + ) + op.create_index('tag_lifetime_end', 'tag', ['lifetime_end'], unique=False) + op.create_index('tag_linked_tag_id', 'tag', ['linked_tag_id'], unique=False) + op.create_index('tag_manifest_list_id', 'tag', ['manifest_list_id'], unique=False) + op.create_index('tag_repository_id', 'tag', ['repository_id'], unique=False) + op.create_index('tag_repository_id_name_hidden', 'tag', ['repository_id', 'name', 'hidden'], unique=False) + op.create_index('tag_repository_id_name_lifetime_end', 'tag', ['repository_id', 'name', 'lifetime_end'], unique=True) + op.create_index('tag_repository_id_name', 'tag', ['repository_id', 'name'], unique=False) + op.create_index('tag_tag_kind_id', 'tag', ['tag_kind_id'], unique=False) + + # ### end Alembic commands ### + + blobplacementlocation_table = table('blobplacementlocation', + column('id', sa.Integer()), + column('name', sa.String()), + ) + + op.bulk_insert( + blobplacementlocation_table, + [ + {'name': 'local_eu'}, + {'name': 'local_us'}, + ], + ) + + tagkind_table = table('tagkind', + column('id', sa.Integer()), + column('name', sa.String()), + ) + + op.bulk_insert( + tagkind_table, + [ + {'id': 1, 'name': 'tag'}, + {'id': 2, 'name': 'release'}, + {'id': 3, 'name': 'channel'}, + ] + ) \ No newline at end of file diff --git a/data/model/repository.py b/data/model/repository.py index c2c47dd6b..cb776db45 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -9,11 +9,11 @@ from cachetools import ttl_cache from data.model import ( config, DataModelException, tag, db_transaction, storage, permission, _basequery) -from data.database import ( - Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag, +from data.database import ( + Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp, - db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind) + db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind, ApprTag) from data.text import prefix_search from util.itertoolrecipes import take @@ -90,8 +90,6 @@ def purge_repository(namespace_name, repository_name): # Delete the repository of all Appr-referenced entries. # Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags, # and MySQL is... particular... about such relationships when deleting. - Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute() - Tag.delete().where(Tag.repository == repo).execute() ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute() ApprTag.delete().where(ApprTag.repository == repo).execute() diff --git a/data/model/storage.py b/data/model/storage.py index 9ab595aff..58d5ce351 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -8,8 +8,8 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn DataModelException, _basequery) from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation, ImageStorageTransformation, ImageStorageSignature, - ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob, - ApprBlob, ensure_under_transaction) + ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, ApprBlob, + ensure_under_transaction) logger = logging.getLogger(__name__) @@ -105,21 +105,15 @@ def garbage_collect_storage(storage_id_whitelist): logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced', is_referenced_checksums) - # Check the ApprBlob tables as well. - query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums)) - blob_referenced_checksums = set([blob.digest for blob in query]) - if blob_referenced_checksums: - logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced', - blob_referenced_checksums) - + # Check the ApprBlob table as well. query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums)) appr_blob_referenced_checksums = set([blob.digest for blob in query]) if appr_blob_referenced_checksums: logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced', appr_blob_referenced_checksums) - unreferenced_checksums = (content_checksums - blob_referenced_checksums - - appr_blob_referenced_checksums - is_referenced_checksums) + unreferenced_checksums = (content_checksums - appr_blob_referenced_checksums - + is_referenced_checksums) # Return all placements for all image storages found not at a CAS path or with a content # checksum that is referenced. diff --git a/data/model/test/test_gc.py b/data/model/test/test_gc.py index d1ea2b72c..d8faa3b16 100644 --- a/data/model/test/test_gc.py +++ b/data/model/test/test_gc.py @@ -10,7 +10,7 @@ from playhouse.test_utils import assert_query_count from data import model, database from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, - Blob, ApprBlob) + ApprBlob) from test.fixtures import * @@ -187,9 +187,6 @@ def assert_gc_integrity(expect_storage_removed=True): if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum)) - for blob_row in Blob.select(): - storage.get_content({preferred}, storage.blob_path(blob_row.digest)) - for blob_row in ApprBlob.select(): storage.get_content({preferred}, storage.blob_path(blob_row.digest)) @@ -592,7 +589,6 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d media_type = database.MediaType.get(name='text/plain') is1 = database.ImageStorage.create(content_checksum=digest, uploading=False) - database.Blob.create(digest=digest, size=0, media_type=media_type) database.ApprBlob.create(digest=digest, size=0, media_type=media_type) location = database.ImageStorageLocation.get(name=preferred) diff --git a/endpoints/appr/models_cnr.py b/endpoints/appr/models_cnr.py index 3c8ccdc6f..19e0d0b9f 100644 --- a/endpoints/appr/models_cnr.py +++ b/endpoints/appr/models_cnr.py @@ -10,7 +10,7 @@ import data.model from app import storage, authentication from data import appr_model from data.database import Repository, MediaType, db_transaction -from data.appr_model.models import OLD_MODELS, NEW_MODELS +from data.appr_model.models import NEW_MODELS from endpoints.appr.models_interface import ( ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface, BlobDescriptor, ChannelView, ChannelReleasesView) @@ -310,5 +310,5 @@ class CNRAppModel(AppRegistryDataInterface): return appr_model.blob.get_blob_locations(digest, self.models_ref) -# Phase 2: Read and write from new tables. +# Phase 3: Read and write from new tables. model = CNRAppModel(NEW_MODELS, features.READONLY_APP_REGISTRY) diff --git a/initdb.py b/initdb.py index cf30aadc7..7184d3cc6 100644 --- a/initdb.py +++ b/initdb.py @@ -14,14 +14,14 @@ from uuid import UUID, uuid4 from threading import Event from email.utils import formatdate -from data.database import (db, all_models, cnr_classes, Role, TeamRole, Visibility, LoginService, +from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService, BuildTriggerService, AccessTokenKind, LogEntryKind, ImageStorageLocation, ImageStorageTransformation, ImageStorageSignatureKind, ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind, QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode, ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind, - RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason, - DeletedNamespace, appr_classes, ApprTagKind, ApprBlobPlacementLocation) + RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes, + ApprTagKind, ApprBlobPlacementLocation) from data import model from data.queue import WorkQueue from app import app, storage as store, tf @@ -361,9 +361,6 @@ def initialize_database(): ImageStorageLocation.create(name='local_eu') ImageStorageLocation.create(name='local_us') - BlobPlacementLocation.create(name='local_eu') - BlobPlacementLocation.create(name='local_us') - ApprBlobPlacementLocation.create(name='local_eu') ApprBlobPlacementLocation.create(name='local_us') @@ -434,10 +431,6 @@ def initialize_database(): RepositoryKind.create(name='image') RepositoryKind.create(name='application') - TagKind.create(name='tag') - TagKind.create(name='release') - TagKind.create(name='channel') - ApprTagKind.create(name='tag') ApprTagKind.create(name='release') ApprTagKind.create(name='channel') @@ -913,7 +906,7 @@ def find_models_missing_data(): # whitelisted. models_missing_data = set() for one_model in all_models: - if one_model in cnr_classes or one_model in appr_classes: + if one_model in appr_classes: continue try: