Move manifest backfill for V1 tags into the new registry model interface

data/registry_model/datatypes.py

@@ -1,6 +1,8 @@
 from enum import Enum, unique
 
 from data.registry_model.datatype import datatype, requiresinput
+from image.docker.schema1 import DockerSchema1Manifest
+
 
 class RepositoryReference(datatype('Repository', [])):
   """ RepositoryReference is a reference to a repository, passed to registry interface methods. """
@@ -68,6 +70,10 @@ class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])):
     """
     return legacy_image
 
+  def get_parsed_manifest(self, validate=True):
+    """ Returns the parsed manifest for this manifest. """
+    return DockerSchema1Manifest(self.manifest_bytes, validate=validate)
+
 
 class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'comment', 'command',
                                            'image_size', 'aggregate_size', 'uploading'])):

data/registry_model/interface.py

@@ -122,3 +122,12 @@ class RegistryDataInterface(object):
   @abstractmethod
   def get_security_status(self, manifest_or_legacy_image):
     """ Returns the security status for the given manifest or legacy image or None if none. """
+
+  @abstractmethod
+  def backfill_manifest_for_tag(self, tag):
+    """ Backfills a manifest for the V1 tag specified.
+        If a manifest already exists for the tag, returns that manifest.
+
+        NOTE: This method will only be necessary until we've completed the backfill, at which point
+        it should be removed.
+    """

data/registry_model/registry_pre_oci_model.py

@@ -2,11 +2,14 @@
 
 from collections import defaultdict
 
+from peewee import IntegrityError
+
 from data import database
 from data import model
 from data.registry_model.interface import RegistryDataInterface
 from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label,
                                            SecurityScanStatus)
+from image.docker.schema1 import DockerSchema1ManifestBuilder
 
 
 class PreOCIModel(RegistryDataInterface):
@@ -199,7 +202,13 @@ class PreOCIModel(RegistryDataInterface):
         model.tag.restore_tag_to_image(repository_ref._db_id, tag_name,
                                        manifest_or_legacy_image.docker_image_id)
 
-    return self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True)
+    # Generate a manifest for the tag, if necessary.
+    tag = self.get_repo_tag(repository_ref, tag_name, include_legacy_image=True)
+    if tag is None:
+      return None
+
+    self.backfill_manifest_for_tag(tag)
+    return tag
 
   def delete_tag(self, repository_ref, tag_name):
     """
@@ -285,5 +294,66 @@ class PreOCIModel(RegistryDataInterface):
 
     return SecurityScanStatus.QUEUED
 
+  def backfill_manifest_for_tag(self, tag):
+    """ Backfills a manifest for the V1 tag specified.
+        If a manifest already exists for the tag, returns that manifest.
+
+        NOTE: This method will only be necessary until we've completed the backfill, at which point
+        it should be removed.
+    """
+    import features
+
+    from app import app, docker_v2_signing_key
+
+    # Ensure that there isn't already a manifest for the tag.
+    tag_manifest = model.tag.get_tag_manifest(tag._db_id)
+    if tag_manifest is not None:
+      return Manifest.for_tag_manifest(tag_manifest)
+
+    # Create the manifest.
+    try:
+      tag_obj = database.RepositoryTag.get(id=tag._db_id)
+    except database.RepositoryTag.DoesNotExist:
+      return None
+
+    repo = tag_obj.repository
+    namespace_name = repo.namespace_user.username
+    repo_name = repo.name
+
+    # Find the v1 metadata for this image and its parents.
+    repo_image = tag_obj.image
+    parents = model.image.get_parent_images(namespace_name, repo_name, repo_image)
+
+    # If the manifest is being generated under the library namespace, then we make its namespace
+    # empty.
+    manifest_namespace = namespace_name
+    if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
+      manifest_namespace = ''
+
+    # Create and populate the manifest builder
+    builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag.name)
+
+    # Add the leaf layer
+    builder.add_layer(repo_image.storage.content_checksum, repo_image.v1_json_metadata)
+
+    for parent_image in parents:
+      builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
+
+    # Sign the manifest with our signing key.
+    manifest = builder.build(docker_v2_signing_key)
+
+    # Write the manifest to the DB.
+    blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo,
+                                                                        manifest.checksums)
+
+    storage_map = {blob.content_checksum: blob.id for blob in blob_query}
+    try:
+      tag_manifest, _ = model.tag.associate_generated_tag_manifest(namespace_name, repo_name,
+                                                                   tag.name, manifest, storage_map)
+    except IntegrityError:
+      tag_manifest = model.tag.get_tag_manifest(tag_obj)
+
+    return Manifest.for_tag_manifest(tag_manifest)
+
 
 pre_oci_model = PreOCIModel()

data/registry_model/test/test_pre_oci_model.py

@@ -4,7 +4,11 @@ import pytest
 
 from playhouse.test_utils import assert_query_count
 
+from app import docker_v2_signing_key
 from data import model
+from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
+                           ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
+                           TagManifestLabel, TagManifest, TagManifestLabel)
 from data.registry_model.registry_pre_oci_model import PreOCIModel
 from data.registry_model.datatypes import RepositoryReference
 
@@ -315,3 +319,47 @@ def test_get_security_status(pre_oci_model):
 
   for tag in tags:
     assert pre_oci_model.get_security_status(tag.legacy_image)
+
+
+@pytest.fixture()
+def clear_rows(initialized_db):
+  # Remove all new-style rows so we can backfill.
+  TagManifestLabelMap.delete().execute()
+  ManifestLabel.delete().execute()
+  ManifestBlob.delete().execute()
+  ManifestLegacyImage.delete().execute()
+  TagManifestToManifest.delete().execute()
+  Manifest.delete().execute()
+  TagManifestLabel.delete().execute()
+  TagManifest.delete().execute()
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('devtable', 'complex'),
+  ('devtable', 'history'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, pre_oci_model):
+  repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  tags = pre_oci_model.list_repository_tags(repository_ref)
+  assert tags
+
+  for tag in tags:
+    assert not tag.manifest_digest
+    assert pre_oci_model.backfill_manifest_for_tag(tag)
+
+  tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True)
+  assert tags
+  for tag in tags:
+    assert tag.manifest_digest
+
+    manifest = pre_oci_model.get_manifest_for_tag(tag)
+    assert manifest
+
+    legacy_image = pre_oci_model.get_legacy_image(repository_ref, tag.legacy_image.docker_image_id,
+                                                  include_parents=True)
+
+    parsed_manifest = manifest.get_parsed_manifest()
+    assert parsed_manifest.leaf_layer_v1_image_id == legacy_image.docker_image_id
+    assert parsed_manifest.parent_image_ids == {p.docker_image_id for p in legacy_image.parents}