Fix backfills of super large manifests by stripping metadata from all but the final layer

This is semantically valid because Docker only uses the leaf layer as the image config when reading a V2_1 manifest

Fixes https://jira.coreos.com/browse/QUAY-1351

image/docker/test/test_schema1.py

@@ -213,3 +213,99 @@ def test_validate_manifest_with_none_metadata_layer(with_key):
   # Ensure the manifest can be reloaded.
   built_bytes = built.bytes.as_encoded_str()
   DockerSchema1Manifest(Bytes.for_string_or_unicode(built_bytes))
+
+
+def test_build_with_metadata_removed():
+  builder = DockerSchema1ManifestBuilder('somenamespace', 'somerepo', 'sometag')
+  builder.add_layer('sha256:abcde', json.dumps({
+    'id': 'someid',
+    'parent': 'someid',
+    'author': u'😱',
+    'comment': 'hello world!',
+    'created': '1975-01-02 12:34',
+    'Size': 5678,
+    'container_config': {
+      'Cmd': 'foobar',
+      'more': 'stuff',
+      'goes': 'here',
+    },
+  }))
+  builder.add_layer('sha256:abcde', json.dumps({
+    'id': 'anotherid',
+    'author': u'😱',
+    'created': '1985-02-03 12:34',
+    'Size': 1234,
+    'container_config': {
+      'Cmd': 'barbaz',
+      'more': 'stuff',
+      'goes': 'here',
+    },
+  }))
+
+  built = builder.build(None)
+  built._validate()
+
+  assert built.leaf_layer_v1_image_id == 'someid'
+
+  with_metadata_removed = builder.with_metadata_removed().build()
+  with_metadata_removed._validate()
+
+  built_layers = list(built.get_layers(None))
+  with_metadata_removed_layers = list(with_metadata_removed.get_layers(None))
+
+  assert len(built_layers) == len(with_metadata_removed_layers)
+  for index, built_layer in enumerate(built_layers):
+    with_metadata_removed_layer = with_metadata_removed_layers[index]
+
+    assert built_layer.layer_id == with_metadata_removed_layer.layer_id
+    assert built_layer.compressed_size == with_metadata_removed_layer.compressed_size
+    assert built_layer.command == with_metadata_removed_layer.command
+    assert built_layer.comment == with_metadata_removed_layer.comment
+    assert built_layer.author == with_metadata_removed_layer.author
+    assert built_layer.blob_digest == with_metadata_removed_layer.blob_digest
+    assert built_layer.created_datetime == with_metadata_removed_layer.created_datetime
+
+  assert built.leaf_layer_v1_image_id == with_metadata_removed.leaf_layer_v1_image_id
+  assert built_layers[-1].layer_id == built.leaf_layer_v1_image_id
+
+  assert (json.loads(built_layers[-1].internal_layer.raw_v1_metadata) ==
+          json.loads(with_metadata_removed_layers[-1].internal_layer.raw_v1_metadata))
+
+
+def test_validate_manifest_without_metadata():
+  test_dir = os.path.dirname(os.path.abspath(__file__))
+  with open(os.path.join(test_dir, 'validated_manifest.json'), 'r') as f:
+    manifest_bytes = f.read()
+
+  manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes), validate=True)
+  digest = manifest.digest
+  assert digest == 'sha256:b5dc4f63fdbd64f34f2314c0747ef81008f9fcddce4edfc3fd0e8ec8b358d571'
+  assert manifest.created_datetime
+
+  with_metadata_removed = manifest._unsigned_builder().with_metadata_removed().build()
+  assert with_metadata_removed.leaf_layer_v1_image_id == manifest.leaf_layer_v1_image_id
+
+  manifest_layers = list(manifest.get_layers(None))
+  with_metadata_removed_layers = list(with_metadata_removed.get_layers(None))
+
+  assert len(manifest_layers) == len(with_metadata_removed_layers)
+  for index, built_layer in enumerate(manifest_layers):
+    with_metadata_removed_layer = with_metadata_removed_layers[index]
+
+    assert built_layer.layer_id == with_metadata_removed_layer.layer_id
+    assert built_layer.compressed_size == with_metadata_removed_layer.compressed_size
+    assert built_layer.command == with_metadata_removed_layer.command
+    assert built_layer.comment == with_metadata_removed_layer.comment
+    assert built_layer.author == with_metadata_removed_layer.author
+    assert built_layer.blob_digest == with_metadata_removed_layer.blob_digest
+    assert built_layer.created_datetime == with_metadata_removed_layer.created_datetime
+
+  assert with_metadata_removed.digest != manifest.digest
+
+  assert with_metadata_removed.namespace == manifest.namespace
+  assert with_metadata_removed.repo_name == manifest.repo_name
+  assert with_metadata_removed.tag == manifest.tag
+  assert with_metadata_removed.created_datetime == manifest.created_datetime
+  assert with_metadata_removed.checksums == manifest.checksums
+  assert with_metadata_removed.image_ids == manifest.image_ids
+  assert with_metadata_removed.parent_image_ids == manifest.parent_image_ids