Add new Manifest, ManifestLabel, ManifestLegacyImage and ManifestBlob tables and start writing and GCing to/from them

This change also starts passing in the manifest interface, rather than the raw data, to the model for writing.

Note that this change does *not* backfill the existing rows in to the new tables; that will occur in a followup PR. The new columns in `tagmanifest` and `tagmanifestlabel` will be used to track the backfill, as it will occur in a worker.

data/database.py

@@ -503,7 +503,8 @@ class User(BaseModel):
                                  TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
                                  RepositoryNotification, OAuthAuthorizationCode,
                                  RepositoryActionCount, TagManifestLabel,
-                                 TeamSync, RepositorySearchScore, DeletedNamespace} | appr_classes
+                                 TeamSync, RepositorySearchScore,
+                                 DeletedNamespace} | appr_classes | v22_classes
       delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
 
 
@@ -651,7 +652,7 @@ class Repository(BaseModel):
     # are cleaned up directly
     skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
                                Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage,
-                               RepositorySearchScore} | appr_classes
+                               RepositorySearchScore} | appr_classes | v22_classes
 
     delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
 
@@ -898,12 +899,6 @@ class RepositoryTag(BaseModel):
     )
 
 
-class TagManifest(BaseModel):
-  tag = ForeignKeyField(RepositoryTag, unique=True)
-  digest = CharField(index=True)
-  json_data = TextField()
-
-
 class BUILD_PHASE(object):
   """ Build phases enum """
   ERROR = 'error'
@@ -1240,21 +1235,6 @@ class Label(BaseModel):
   source_type = ForeignKeyField(LabelSourceType)
 
 
-class TagManifestLabel(BaseModel):
-  """ Mapping from a tag manifest to a label.
-  """
-  repository = ForeignKeyField(Repository, index=True)
-  annotated = ForeignKeyField(TagManifest, index=True)
-  label = ForeignKeyField(Label)
-
-  class Meta:
-    database = db
-    read_slaves = (read_slave,)
-    indexes = (
-      (('annotated', 'label'), True),
-    )
-
-
 class ApprBlob(BaseModel):
   """ ApprBlob represents a content-addressable object stored outside of the database.
   """
@@ -1385,10 +1365,104 @@ class AppSpecificAuthToken(BaseModel):
     indexes = (
       (('user', 'expiration'), False),
     )
-  
+
+
+class Manifest(BaseModel):
+  """ Manifest represents a single manifest under a repository. Within a repository,
+      there can only be one manifest with the same digest.
+  """
+  repository = ForeignKeyField(Repository)
+  digest = CharField(index=True)
+  media_type = EnumField(MediaType)
+  manifest_bytes = TextField()
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('repository', 'digest'), True),
+      (('repository', 'media_type'), False),
+    )
+
+
+class ManifestLabel(BaseModel):
+  """ ManifestLabel represents a label applied to a Manifest, within a repository.
+      Note that since Manifests are stored per-repository, the repository here is
+      a bit redundant, but we do so to make cleanup easier.
+  """
+  repository = ForeignKeyField(Repository, index=True)
+  manifest = ForeignKeyField(Manifest)
+  label = ForeignKeyField(Label)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('manifest', 'label'), True),
+    )
+
+
+class ManifestBlob(BaseModel):
+  """ ManifestBlob represents a blob that is used by a manifest. """
+  repository = ForeignKeyField(Repository, index=True)
+  manifest = ForeignKeyField(Manifest)
+  blob = ForeignKeyField(ImageStorage)
+  blob_index = IntegerField() # 0-indexed location of the blob in the manifest.
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('manifest', 'blob'), True),
+      (('manifest', 'blob_index'), True),
+    )
+
+
+class ManifestLegacyImage(BaseModel):
+  """ For V1-compatible manifests only, this table maps from the manifest to its associated
+      Docker image.
+  """
+  repository = ForeignKeyField(Repository, index=True)
+  manifest = ForeignKeyField(Manifest, unique=True)
+  image = ForeignKeyField(Image)
+
+
+class TagManifest(BaseModel):
+  """ TO BE DEPRECATED: The manifest for a tag. """
+  tag = ForeignKeyField(RepositoryTag, unique=True)
+  digest = CharField(index=True)
+  json_data = TextField()
+
+  # Note: `manifest` will be back-filled by a worker and may not be present
+  # currently.
+  manifest = ForeignKeyField(Manifest, null=True, index=True)
+  broken = BooleanField(null=True, index=True)
+
+class TagManifestLabel(BaseModel):
+  """ TO BE DEPRECATED: Mapping from a tag manifest to a label.
+  """
+  repository = ForeignKeyField(Repository, index=True)
+  annotated = ForeignKeyField(TagManifest, index=True)
+  label = ForeignKeyField(Label)
+
+  # Note: `manifest_label` will be back-filled by a worker and may not be present
+  # currently.
+  manifest_label = ForeignKeyField(ManifestLabel, null=True, index=True)
+  broken_manifest = BooleanField(null=True, index=True)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('annotated', 'label'), True),
+    )
+
+
 
 appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
                     ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
                     ApprBlobPlacement])
+v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage])
+
 is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
 all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]