Merge pull request #3378 from quay/joseph.schorr/QUAY-1351/fix-super-large-manifest-backfill

Fix backfills of super large manifests by stripping metadata from all but the final layer

data/registry_model/registry_pre_oci_model.py

@@ -473,8 +473,6 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
     assert not tag_obj.hidden
 
     repo = tag_obj.repository
-    namespace_name = repo.namespace_user.username
-    repo_name = repo.name
 
     # Write the manifest to the DB.
     manifest = self._build_manifest_for_legacy_image(tag_obj.name, tag_obj.image)

data/registry_model/shared.py

@@ -17,6 +17,8 @@ from image.docker.schema2 import EMPTY_LAYER_BLOB_DIGEST
 
 logger = logging.getLogger(__name__)
 
+# The maximum size for generated manifest after which we remove extra metadata.
+MAXIMUM_GENERATED_MANIFEST_SIZE = 3 * 1024 * 1024 # 3 MB
 
 class SharedModel:
   """
@@ -445,9 +447,19 @@ class SharedModel:
 
       builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
 
-    # Sign the manifest with our signing key.
     try:
-      return builder.build(docker_v2_signing_key)
+      built_manifest = builder.build(docker_v2_signing_key)
+
+      # If the generated manifest is greater than the maximum size, regenerate it with
+      # intermediate metadata layers stripped down to their bare essentials.
+      if len(built_manifest.bytes.as_encoded_str()) > MAXIMUM_GENERATED_MANIFEST_SIZE:
+        built_manifest = builder.with_metadata_removed().build(docker_v2_signing_key)
+
+      if len(built_manifest.bytes.as_encoded_str()) > MAXIMUM_GENERATED_MANIFEST_SIZE:
+        logger.error('Legacy image is too large to generate manifest')
+        return None
+
+      return built_manifest
     except ManifestException as me:
       logger.exception('Got exception when trying to build manifest for legacy image %s',
                        legacy_image_row)