Merge pull request #3148 from quay/joseph.schorr/QUAY-950/appr-step-3

Phase 3 of Appr migration

data/appr_model/models.py

@@ -1,7 +1,5 @@
 from collections import namedtuple
 
-from data.database import (Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
-                           ManifestListManifest, Manifest, BlobPlacement, Channel)
 from data.database import (ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
                            ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
                            ApprBlobPlacement, ApprChannel)
@@ -11,10 +9,6 @@ ModelsRef = namedtuple('ModelsRef', ['Tag', 'TagKind', 'BlobPlacementLocation',
                                      'BlobPlacement', 'Channel', 'manifestlistmanifest_set_name',
                                      'tag_set_prefetch_name'])
 
-OLD_MODELS = ModelsRef(Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
-                       ManifestListManifest, Manifest, BlobPlacement, Channel,
-                       'manifestlistmanifest_set', 'tag_set_prefetch')
-
 NEW_MODELS = ModelsRef(ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
                        ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
                        ApprBlobPlacement, ApprChannel, 'apprmanifestlistmanifest_set',

data/database.py

@@ -471,7 +471,7 @@ class User(BaseModel):
                                  TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
                                  RepositoryNotification, OAuthAuthorizationCode,
                                  RepositoryActionCount, TagManifestLabel,
-                                 TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes | appr_classes
+                                 TeamSync, RepositorySearchScore, DeletedNamespace} | appr_classes
       delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
 
 
@@ -619,7 +619,7 @@ class Repository(BaseModel):
     # are cleaned up directly
     skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
                                Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage,
-                               RepositorySearchScore} | cnr_classes | appr_classes
+                               RepositorySearchScore} | appr_classes
 
     delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
 
@@ -1231,17 +1231,6 @@ class TagManifestLabel(BaseModel):
     )
 
 
-class Blob(BaseModel):
-  """ Blob represents a content-addressable object stored outside of the database.
-      This is deprecated in favor of ApprBlob.
-  """
-  digest = CharField(index=True, unique=True)
-  media_type = EnumField(MediaType)
-  size = BigIntegerField()
-  uncompressed_size = BigIntegerField(null=True)
-
-
-
 class ApprBlob(BaseModel):
   """ ApprBlob represents a content-addressable object stored outside of the database.
   """
@@ -1251,34 +1240,12 @@ class ApprBlob(BaseModel):
   uncompressed_size = BigIntegerField(null=True)
 
 
-class BlobPlacementLocation(BaseModel):
-  """ BlobPlacementLocation is an enumeration of the possible storage locations for Blobs.
-      This is deprecated in favor of ApprBlobPlacementLocation.
-  """
-  name = CharField(index=True, unique=True)
-
-
 class ApprBlobPlacementLocation(BaseModel):
   """ ApprBlobPlacementLocation is an enumeration of the possible storage locations for ApprBlobs.
   """
   name = CharField(index=True, unique=True)
 
 
-class BlobPlacement(BaseModel):
-  """ BlobPlacement represents the location of a Blob.
-      This is deprecated in favor of ApprBlobPlacement.
-  """
-  blob = ForeignKeyField(Blob)
-  location = EnumField(BlobPlacementLocation)
-
-  class Meta:
-    database = db
-    read_slaves = (read_slave,)
-    indexes = (
-      (('blob', 'location'), True),
-    )
-
-
 class ApprBlobPlacement(BaseModel):
   """ ApprBlobPlacement represents the location of a Blob.
   """
@@ -1293,15 +1260,6 @@ class ApprBlobPlacement(BaseModel):
     )
 
 
-class Manifest(BaseModel):
-  """ Manifest represents the metadata and collection of blobs that comprise a container image.
-      This is deprecated in favor of ApprManifest.
-  """
-  digest = CharField(index=True, unique=True)
-  media_type = EnumField(MediaType)
-  manifest_json = JSONField()
-
-
 class ApprManifest(BaseModel):
   """ ApprManifest represents the metadata and collection of blobs that comprise an Appr image.
   """
@@ -1310,21 +1268,6 @@ class ApprManifest(BaseModel):
   manifest_json = JSONField()
 
 
-class ManifestBlob(BaseModel):
-  """ ManifestBlob is a many-to-many relation table linking Manifests and Blobs.
-      This is deprecated in favor of ApprManifestBlob.
-  """
-  manifest = ForeignKeyField(Manifest, index=True)
-  blob = ForeignKeyField(Blob, index=True)
-
-  class Meta:
-    database = db
-    read_slaves = (read_slave,)
-    indexes = (
-      (('manifest', 'blob'), True),
-    )
-
-
 class ApprManifestBlob(BaseModel):
   """ ApprManifestBlob is a many-to-many relation table linking ApprManifests and ApprBlobs.
   """
@@ -1339,16 +1282,6 @@ class ApprManifestBlob(BaseModel):
     )
 
 
-class ManifestList(BaseModel):
-  """ ManifestList represents all of the various manifests that compose a Tag.
-      This is deprecated in favor of ApprManifestList.
-  """
-  digest = CharField(index=True, unique=True)
-  manifest_list_json = JSONField()
-  schema_version = CharField()
-  media_type = EnumField(MediaType)
-
-
 class ApprManifestList(BaseModel):
   """ ApprManifestList represents all of the various Appr manifests that compose an ApprTag.
   """
@@ -1358,45 +1291,12 @@ class ApprManifestList(BaseModel):
   media_type = EnumField(MediaType)
 
 
-class TagKind(BaseModel):
-  """ TagKind is a enumtable to reference tag kinds.
-      This model is deprecated in favor of ApprTagKind.
-  """
-  name = CharField(index=True, unique=True)
-
 
 class ApprTagKind(BaseModel):
   """ ApprTagKind is a enumtable to reference tag kinds.
   """
   name = CharField(index=True, unique=True)
 
-
-class Tag(BaseModel):
-  """ Tag represents a user-facing alias for referencing a ManifestList.
-      This model is deprecated in favor of ApprTag.
-  """
-  name = CharField()
-  repository = ForeignKeyField(Repository)
-  manifest_list = ForeignKeyField(ManifestList, null=True)
-  lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
-  lifetime_end = BigIntegerField(null=True, index=True)
-  hidden = BooleanField(default=False)
-  reverted = BooleanField(default=False)
-  protected = BooleanField(default=False)
-  tag_kind = EnumField(TagKind)
-  linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
-
-  class Meta:
-    database = db
-    read_slaves = (read_slave,)
-    indexes = (
-      (('repository', 'name'), False),
-      (('repository', 'name', 'hidden'), False),
-      # This unique index prevents deadlocks when concurrently moving and deleting tags
-      (('repository', 'name', 'lifetime_end'), True),
-    )
-
-
 class ApprTag(BaseModel):
   """ ApprTag represents a user-facing alias for referencing an ApprManifestList.
   """
@@ -1421,30 +1321,9 @@ class ApprTag(BaseModel):
       (('repository', 'name', 'lifetime_end'), True),
     )
 
-Channel = Tag.alias()
 ApprChannel = ApprTag.alias()
 
 
-class ManifestListManifest(BaseModel):
-  """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests.
-      This model is deprecated in favor of ApprManifestListManifest.
-  """
-  manifest_list = ForeignKeyField(ManifestList, index=True)
-  manifest = ForeignKeyField(Manifest, index=True)
-  operating_system = CharField(null=True)
-  architecture = CharField(null=True)
-  platform_json = JSONField(null=True)
-  media_type = EnumField(MediaType)
-
-  class Meta:
-    database = db
-    read_slaves = (read_slave,)
-    indexes = (
-      (('manifest_list', 'operating_system', 'architecture', 'media_type'), False),
-      (('manifest_list', 'media_type'), False),
-    )
-
-
 class ApprManifestListManifest(BaseModel):
   """ ApprManifestListManifest is a many-to-many relation table linking ApprManifestLists and
       ApprManifests.
@@ -1484,8 +1363,6 @@ class AppSpecificAuthToken(BaseModel):
     )
   
 
-cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
-                   ManifestListManifest, Manifest, BlobPlacement])
 appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
                     ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
                     ApprBlobPlacement])

data/migrations/versions/7a525c68eb13_add_oci_app_models.py

@@ -13,6 +13,7 @@ down_revision = 'e2894a3a3c19'
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import mysql
+from sqlalchemy.sql import table, column
 from util.migrate import UTF8LongText, UTF8CharField
 
 
@@ -274,8 +275,13 @@ def upgrade(tables, tester):
   )
   op.create_index('manifestlayerscan_layer_id', 'manifestlayerscan', ['layer_id'], unique=True)
 
+  blobplacementlocation_table = table('blobplacementlocation',
+      column('id', sa.Integer()),
+      column('name', sa.String()),
+  )
+
   op.bulk_insert(
-    tables.blobplacementlocation,
+    blobplacementlocation_table,
     [
       {'name': 'local_eu'},
       {'name': 'local_us'},
@@ -297,8 +303,13 @@ def upgrade(tables, tester):
     ],
   )
 
+  tagkind_table = table('tagkind',
+      column('id', sa.Integer()),
+      column('name', sa.String()),
+  )
+
   op.bulk_insert(
-    tables.tagkind,
+    tagkind_table,
     [
       {'id': 1, 'name': 'tag'},
       {'id': 2, 'name': 'release'},
@@ -306,7 +317,6 @@ def upgrade(tables, tester):
     ]
   )
 
-
 def downgrade(tables, tester):
   op.drop_table('manifestlayerscan')
   op.drop_table('manifestlayerdockerv1')

data/migrations/versions/d17c695859ea_delete_old_appr_tables.py

@@ -0,0 +1,189 @@
+"""Delete old Appr tables
+
+Revision ID: d17c695859ea
+Revises: 5d463ea1e8a8
+Create Date: 2018-07-16 15:21:11.593040
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'd17c695859ea'
+down_revision = '5d463ea1e8a8'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import table, column
+from util.migrate import UTF8LongText, UTF8CharField
+
+def upgrade(tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('tag')
+    op.drop_table('manifestlistmanifest')
+    op.drop_table('manifestlist')
+    op.drop_table('manifestblob')
+    op.drop_table('manifest')
+    op.drop_table('blobplacement')
+    op.drop_table('blob')
+    op.drop_table('blobplacementlocation')
+    op.drop_table('tagkind')
+    # ### end Alembic commands ###
+
+
+def downgrade(tables, tester):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'tagkind',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_tagkind'))
+    )
+    op.create_index('tagkind_name', 'tagkind', ['name'], unique=True)
+
+    op.create_table(
+        'blobplacementlocation',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacementlocation'))
+    )
+    op.create_index('blobplacementlocation_name', 'blobplacementlocation', ['name'], unique=True)
+
+    op.create_table(
+        'blob',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('digest', sa.String(length=255), nullable=False),
+        sa.Column('media_type_id', sa.Integer(), nullable=False),
+        sa.Column('size', sa.BigInteger(), nullable=False),
+        sa.Column('uncompressed_size', sa.BigInteger(), nullable=True),
+        sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_blob_media_type_id_mediatype')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_blob'))
+    )
+    op.create_index('blob_digest', 'blob', ['digest'], unique=True)
+    op.create_index('blob_media_type_id', 'blob', ['media_type_id'], unique=False)
+
+    op.create_table(
+        'manifest',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('digest', sa.String(length=255), nullable=False),
+        sa.Column('media_type_id', sa.Integer(), nullable=False),
+        sa.Column('manifest_json', UTF8LongText, nullable=False),
+        sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifest_media_type_id_mediatype')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_manifest'))
+    )
+    op.create_index('manifest_digest', 'manifest', ['digest'], unique=True)
+    op.create_index('manifest_media_type_id', 'manifest', ['media_type_id'], unique=False)
+
+    op.create_table(
+        'manifestlist',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('digest', sa.String(length=255), nullable=False),
+        sa.Column('manifest_list_json', UTF8LongText, nullable=False),
+        sa.Column('schema_version', UTF8CharField(length=255), nullable=False),
+        sa.Column('media_type_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlist_media_type_id_mediatype')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlist'))
+    )
+    op.create_index('manifestlist_digest', 'manifestlist', ['digest'], unique=True)
+    op.create_index('manifestlist_media_type_id', 'manifestlist', ['media_type_id'], unique=False)
+
+    op.create_table(
+        'blobplacement',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('blob_id', sa.Integer(), nullable=False),
+        sa.Column('location_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_blobplacement_blob_id_blob')),
+        sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobplacement_location_id_blobplacementlocation')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacement'))
+    )
+    op.create_index('blobplacement_blob_id', 'blobplacement', ['blob_id'], unique=False)
+    op.create_index('blobplacement_blob_id_location_id', 'blobplacement', ['blob_id', 'location_id'], unique=True)
+    op.create_index('blobplacement_location_id', 'blobplacement', ['location_id'], unique=False)
+
+    op.create_table(
+        'manifestblob',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('manifest_id', sa.Integer(), nullable=False),
+        sa.Column('blob_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_manifestblob_blob_id_blob')),
+        sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestblob_manifest_id_manifest')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestblob'))
+    )
+    op.create_index('manifestblob_blob_id', 'manifestblob', ['blob_id'], unique=False)
+    op.create_index('manifestblob_manifest_id', 'manifestblob', ['manifest_id'], unique=False)
+    op.create_index('manifestblob_manifest_id_blob_id', 'manifestblob', ['manifest_id', 'blob_id'], unique=True)
+
+    op.create_table(
+        'manifestlistmanifest',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('manifest_list_id', sa.Integer(), nullable=False),
+        sa.Column('manifest_id', sa.Integer(), nullable=False),
+        sa.Column('operating_system', UTF8CharField(length=255), nullable=True),
+        sa.Column('architecture', UTF8CharField(length=255), nullable=True),
+        sa.Column('platform_json', UTF8LongText, nullable=True),
+        sa.Column('media_type_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestlistmanifest_manifest_id_manifest')),
+        sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_manifestlistmanifest_manifest_list_id_manifestlist')),
+        sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlistmanifest_media_type_id_mediatype')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlistmanifest'))
+    )
+    op.create_index('manifestlistmanifest_manifest_id', 'manifestlistmanifest', ['manifest_id'], unique=False)
+    op.create_index('manifestlistmanifest_manifest_list_id', 'manifestlistmanifest', ['manifest_list_id'], unique=False)
+    op.create_index('manifestlistmanifest_manifest_listid_os_arch_mtid', 'manifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False)
+    op.create_index('manifestlistmanifest_manifest_listid_mtid', 'manifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False)
+    op.create_index('manifestlistmanifest_media_type_id', 'manifestlistmanifest', ['media_type_id'], unique=False)
+
+    op.create_table(
+        'tag',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', UTF8CharField(length=190), nullable=False),
+        sa.Column('repository_id', sa.Integer(), nullable=False),
+        sa.Column('manifest_list_id', sa.Integer(), nullable=True),
+        sa.Column('lifetime_start', sa.BigInteger(), nullable=False),
+        sa.Column('lifetime_end', sa.BigInteger(), nullable=True),
+        sa.Column('hidden', sa.Boolean(), nullable=False),
+        sa.Column('reverted', sa.Boolean(), nullable=False),
+        sa.Column('protected', sa.Boolean(), nullable=False),
+        sa.Column('tag_kind_id', sa.Integer(), nullable=False),
+        sa.Column('linked_tag_id', sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(['linked_tag_id'], ['tag.id'], name=op.f('fk_tag_linked_tag_id_tag')),
+        sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_tag_manifest_list_id_manifestlist')),
+        sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tag_repository_id_repository')),
+        sa.ForeignKeyConstraint(['tag_kind_id'], ['tagkind.id'], name=op.f('fk_tag_tag_kind_id_tagkind')),
+        sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
+    )
+    op.create_index('tag_lifetime_end', 'tag', ['lifetime_end'], unique=False)
+    op.create_index('tag_linked_tag_id', 'tag', ['linked_tag_id'], unique=False)
+    op.create_index('tag_manifest_list_id', 'tag', ['manifest_list_id'], unique=False)
+    op.create_index('tag_repository_id', 'tag', ['repository_id'], unique=False)
+    op.create_index('tag_repository_id_name_hidden', 'tag', ['repository_id', 'name', 'hidden'], unique=False)
+    op.create_index('tag_repository_id_name_lifetime_end', 'tag', ['repository_id', 'name', 'lifetime_end'], unique=True)
+    op.create_index('tag_repository_id_name', 'tag', ['repository_id', 'name'], unique=False)
+    op.create_index('tag_tag_kind_id', 'tag', ['tag_kind_id'], unique=False)
+
+    # ### end Alembic commands ###
+
+    blobplacementlocation_table = table('blobplacementlocation',
+        column('id', sa.Integer()),
+        column('name', sa.String()),
+    )
+
+    op.bulk_insert(
+        blobplacementlocation_table,
+        [
+        {'name': 'local_eu'},
+        {'name': 'local_us'},
+        ],
+    )
+
+    tagkind_table = table('tagkind',
+        column('id', sa.Integer()),
+        column('name', sa.String()),
+    )
+
+    op.bulk_insert(
+        tagkind_table,
+        [
+        {'id': 1, 'name': 'tag'},
+        {'id': 2, 'name': 'release'},
+        {'id': 3, 'name': 'channel'},
+        ]
+    )

data/model/repository.py

@@ -9,11 +9,11 @@ from cachetools import ttl_cache
 
 from data.model import (
   config, DataModelException, tag, db_transaction, storage, permission, _basequery)
-from data.database import (
+from data.database import ( 
-  Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag,
+  Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
   RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
   DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
-  db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind)
+  db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind, ApprTag)
 from data.text import prefix_search
 from util.itertoolrecipes import take
 
@@ -90,8 +90,6 @@ def purge_repository(namespace_name, repository_name):
   # Delete the repository of all Appr-referenced entries.
   # Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
   # and MySQL is... particular... about such relationships when deleting.
-  Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute()
-  Tag.delete().where(Tag.repository == repo).execute()
   ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute()
   ApprTag.delete().where(ApprTag.repository == repo).execute()
 

data/model/storage.py

@@ -8,8 +8,8 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
                         DataModelException, _basequery)
 from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
                            ImageStorageTransformation, ImageStorageSignature,
-                           ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob,
+                           ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, ApprBlob,
-                           ApprBlob, ensure_under_transaction)
+                           ensure_under_transaction)
 
 
 logger = logging.getLogger(__name__)
@@ -105,21 +105,15 @@ def garbage_collect_storage(storage_id_whitelist):
           logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced',
                          is_referenced_checksums)
 
-        # Check the ApprBlob tables as well.
+        # Check the ApprBlob table as well.
-        query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums))
-        blob_referenced_checksums = set([blob.digest for blob in query])
-        if blob_referenced_checksums:
-          logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced',
-                         blob_referenced_checksums)
-
         query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums))
         appr_blob_referenced_checksums = set([blob.digest for blob in query])
         if appr_blob_referenced_checksums:
           logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced',
                          appr_blob_referenced_checksums)
 
-        unreferenced_checksums = (content_checksums - blob_referenced_checksums -
+        unreferenced_checksums = (content_checksums - appr_blob_referenced_checksums -
-                                  appr_blob_referenced_checksums - is_referenced_checksums)
+                                  is_referenced_checksums)
 
       # Return all placements for all image storages found not at a CAS path or with a content
       # checksum that is referenced.

data/model/test/test_gc.py

@@ -10,7 +10,7 @@ from playhouse.test_utils import assert_query_count
 
 from data import model, database
 from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
-                           Blob, ApprBlob)
+                           ApprBlob)
 from test.fixtures import *
 
 
@@ -187,9 +187,6 @@ def assert_gc_integrity(expect_storage_removed=True):
     if storage_row.cas_path:
       storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
 
-  for blob_row in Blob.select():
-    storage.get_content({preferred}, storage.blob_path(blob_row.digest))
-
   for blob_row in ApprBlob.select():
     storage.get_content({preferred}, storage.blob_path(blob_row.digest))
 
@@ -592,7 +589,6 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d
     media_type = database.MediaType.get(name='text/plain')
 
     is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
-    database.Blob.create(digest=digest, size=0, media_type=media_type)
     database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
 
     location = database.ImageStorageLocation.get(name=preferred)

endpoints/appr/models_cnr.py

@@ -10,7 +10,7 @@ import data.model
 from app import storage, authentication
 from data import appr_model
 from data.database import Repository, MediaType, db_transaction
-from data.appr_model.models import OLD_MODELS, NEW_MODELS
+from data.appr_model.models import NEW_MODELS
 from endpoints.appr.models_interface import (
   ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface,
   BlobDescriptor, ChannelView, ChannelReleasesView)
@@ -310,5 +310,5 @@ class CNRAppModel(AppRegistryDataInterface):
     return appr_model.blob.get_blob_locations(digest, self.models_ref)
 
 
-# Phase 2: Read and write from new tables.
+# Phase 3: Read and write from new tables.
 model = CNRAppModel(NEW_MODELS, features.READONLY_APP_REGISTRY)

initdb.py

@@ -14,14 +14,14 @@ from uuid import UUID, uuid4
 from threading import Event
 
 from email.utils import formatdate
-from data.database import (db, all_models, cnr_classes, Role, TeamRole, Visibility, LoginService,
+from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService,
                            BuildTriggerService, AccessTokenKind, LogEntryKind, ImageStorageLocation,
                            ImageStorageTransformation, ImageStorageSignatureKind,
                            ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind,
                            QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
                            ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
-                           RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason,
+                           RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes,
-                           DeletedNamespace, appr_classes, ApprTagKind, ApprBlobPlacementLocation)
+                           ApprTagKind, ApprBlobPlacementLocation)
 from data import model
 from data.queue import WorkQueue
 from app import app, storage as store, tf
@@ -361,9 +361,6 @@ def initialize_database():
   ImageStorageLocation.create(name='local_eu')
   ImageStorageLocation.create(name='local_us')
 
-  BlobPlacementLocation.create(name='local_eu')
-  BlobPlacementLocation.create(name='local_us')
-
   ApprBlobPlacementLocation.create(name='local_eu')
   ApprBlobPlacementLocation.create(name='local_us')
 
@@ -434,10 +431,6 @@ def initialize_database():
   RepositoryKind.create(name='image')
   RepositoryKind.create(name='application')
 
-  TagKind.create(name='tag')
-  TagKind.create(name='release')
-  TagKind.create(name='channel')
-
   ApprTagKind.create(name='tag')
   ApprTagKind.create(name='release')
   ApprTagKind.create(name='channel')
@@ -913,7 +906,7 @@ def find_models_missing_data():
   # whitelisted.
   models_missing_data = set()
   for one_model in all_models:
-    if one_model in cnr_classes or one_model in appr_classes:
+    if one_model in appr_classes:
       continue
 
     try: