Add support for pushing and pulling schema 2 manifests with remote layers

This is required for windows image support

image/docker/schema2/test/test_list.py

@@ -28,7 +28,7 @@ MANIFESTLIST_BYTES = json.dumps({
   "manifests": [
     {
       "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-      "size": 983,
+      "size": 946,
       "digest": "sha256:e6",
       "platform": {
         "architecture": "ppc64le",
@@ -56,7 +56,7 @@ NO_AMD_MANIFESTLIST_BYTES = json.dumps({
   "manifests": [
     {
       "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-      "size": 983,
+      "size": 946,
       "digest": "sha256:e6",
       "platform": {
         "architecture": "ppc64le",
@@ -76,7 +76,7 @@ def test_valid_manifestlist():
   manifestlist = DockerSchema2ManifestList(MANIFESTLIST_BYTES)
   assert len(manifestlist.manifests(_get_manifest)) == 2
   assert (manifestlist.digest ==
-          'sha256:7e22fdbe49736329786c9b4fdc154cc9251b190ca6b4cf33aed00efc0fc3df25')
+          'sha256:340d7dadea77035533a2d43e8ff711ecca1965978a6e7699b87e32b35f76038d')
 
   assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
   assert manifestlist.bytes == MANIFESTLIST_BYTES
@@ -109,7 +109,7 @@ def test_get_v1_compatible_manifest_no_matching_list():
   manifestlist = DockerSchema2ManifestList(NO_AMD_MANIFESTLIST_BYTES)
   assert len(manifestlist.manifests(_get_manifest)) == 1
   assert (manifestlist.digest ==
-          'sha256:50150251101420a020ab4a3e77e9d167a18b09bd4eeb0cc65e0eafab95cf79cf')
+          'sha256:40ed1cfe692333bfa519a9bfed9676975a990fff5afd35efa628320c39c793ca')
 
   assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
   assert manifestlist.bytes == NO_AMD_MANIFESTLIST_BYTES

image/docker/schema2/test/test_manifest.py

@@ -31,6 +31,38 @@ MANIFEST_BYTES = json.dumps({
     "size": 1885,
     "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
   },
+  "layers": [
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 1234,
+      "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736",
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 32654,
+      "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 16724,
+      "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 73109,
+      "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736"
+    },
+  ],
+})
+
+REMOTE_MANIFEST_BYTES = json.dumps({
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+  "config": {
+    "mediaType": "application/vnd.docker.container.image.v1+json",
+    "size": 1885,
+    "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
+  },
   "layers": [
     {
       "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
@@ -61,12 +93,12 @@ def test_valid_manifest():
   assert manifest.config.size == 1885
   assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
   assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
+  assert not manifest.has_remote_layer
 
   assert len(manifest.layers) == 4
-  assert manifest.layers[0].is_remote
   assert manifest.layers[0].compressed_size == 1234
   assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
-  assert manifest.layers[0].urls
+  assert not manifest.layers[0].is_remote
 
   assert manifest.leaf_layer == manifest.layers[3]
   assert not manifest.leaf_layer.is_remote
@@ -75,6 +107,37 @@ def test_valid_manifest():
   blob_digests = list(manifest.blob_digests)
   expected = [str(layer.digest) for layer in manifest.layers] + [manifest.config.digest]
   assert blob_digests == expected
+  assert list(manifest.local_blob_digests) == expected
+
+
+def test_valid_remote_manifest():
+  manifest = DockerSchema2Manifest(REMOTE_MANIFEST_BYTES)
+  assert manifest.config.size == 1885
+  assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
+  assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
+  assert manifest.has_remote_layer
+
+  assert len(manifest.layers) == 4
+  assert manifest.layers[0].compressed_size == 1234
+  assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
+  assert manifest.layers[0].is_remote
+  assert manifest.layers[0].urls == ['http://some/url']
+
+  assert manifest.leaf_layer == manifest.layers[3]
+  assert not manifest.leaf_layer.is_remote
+  assert manifest.leaf_layer.compressed_size == 73109
+
+  expected = set([str(layer.digest) for layer in manifest.layers] + [manifest.config.digest])
+
+  blob_digests = set(manifest.blob_digests)
+  local_digests = set(manifest.local_blob_digests)
+
+  assert blob_digests == expected
+  assert local_digests == (expected - {manifest.layers[0].digest})
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.legacy_image_ids is None
 
 
 def test_schema2_builder():
@@ -110,6 +173,7 @@ def test_get_manifest_labels():
 
 def test_build_schema1():
   manifest = DockerSchema2Manifest(MANIFEST_BYTES)
+  assert not manifest.has_remote_layer
 
   builder = DockerSchema1ManifestBuilder('somenamespace', 'somename', 'sometag')
   manifest.populate_schema1_builder(builder, lambda digest: CONFIG_BYTES)
@@ -209,3 +273,22 @@ def test_generate_legacy_layers():
   assert legacy_layers[0].parent_image_id is None
 
   assert legacy_layers[0].image_id != legacy_layers[1]
+
+
+def test_remote_layer_manifest():
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest('sha256:abcd', 1234)
+  builder.add_layer('sha256:adef', 1234, urls=['http://some/url'])
+  builder.add_layer('sha256:1352', 4567)
+  builder.add_layer('sha256:1353', 4567)
+  manifest = builder.build()
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.legacy_image_ids is None
+
+  schema1 = manifest.get_v1_compatible_manifest('somenamespace', 'somename', 'sometag', None)
+  assert schema1 is None
+
+  assert set(manifest.blob_digests) == {'sha256:adef', 'sha256:abcd', 'sha256:1352', 'sha256:1353'}
+  assert set(manifest.local_blob_digests) == {'sha256:abcd', 'sha256:1352', 'sha256:1353'}