Add support for pushing and pulling schema 2 manifests with remote layers

This is required for windows image support

image/docker/interfaces.py

@@ -53,6 +53,13 @@ class ManifestInterface(object):
         config as a blob, the blob will be included here.
     """
 
+  @abstractproperty
+  def local_blob_digests(self):
+    """ Returns an iterator over all the *non-remote* blob digests referenced by this manifest,
+        from base to leaf. The blob digests are strings with prefixes. For manifests that reference
+        config as a blob, the blob will be included here.
+    """
+
   @abstractmethod
   def child_manifests(self, lookup_manifest_fn):
     """ Returns an iterator of all manifests that live under this manifest, if any or None if not

image/docker/schema1.py

@@ -73,7 +73,7 @@ class InvalidSchema1Signature(ManifestException):
 
 
 class Schema1Layer(namedtuple('Schema1Layer', ['digest', 'v1_metadata', 'raw_v1_metadata',
-                                               'compressed_size'])):
+                                               'compressed_size', 'is_remote'])):
   """
   Represents all of the data about an individual layer in a given Manifest.
   This is the union of the fsLayers (digest) and the history entries (v1_compatibility).
@@ -302,6 +302,10 @@ class DockerSchema1Manifest(ManifestInterface):
   def blob_digests(self):
     return [str(layer.digest) for layer in self.layers]
 
+  @property
+  def local_blob_digests(self):
+    return self.blob_digests
+
   def child_manifests(self, lookup_manifest_fn):
     return None
 
@@ -349,7 +353,7 @@ class DockerSchema1Manifest(ManifestInterface):
                                     command, labels)
 
       compressed_size = v1_metadata.get('Size')
-      yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size)
+      yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size, False)
 
   @property
   def _payload(self):

image/docker/schema2/list.py

@@ -224,6 +224,10 @@ class DockerSchema2ManifestList(ManifestInterface):
     manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]
     return [m[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY] for m in manifests]
 
+  @property
+  def local_blob_digests(self):
+    return self.blob_digests
+
   @lru_cache(maxsize=1)
   def manifests(self, lookup_manifest_fn):
     """ Returns the manifests in the list. The `lookup_manifest_fn` is a function

image/docker/schema2/manifest.py

@@ -136,6 +136,10 @@ class DockerSchema2Manifest(ManifestInterface):
     except ValidationError as ve:
       raise MalformedSchema2Manifest('manifest data does not match schema: %s' % ve)
 
+    for layer in self.layers:
+      if layer.is_remote and not layer.urls:
+        raise MalformedSchema2Manifest('missing `urls` for remote layer')
+
   @property
   def schema_version(self):
     return 2
@@ -169,18 +173,39 @@ class DockerSchema2Manifest(ManifestInterface):
   def leaf_layer(self):
     return self.layers[-1]
 
+  @property
+  def has_remote_layer(self):
+    for layer in self.layers:
+      if layer.is_remote:
+        return True
+
+    return False
+
   @property
   def leaf_layer_v1_image_id(self):
+    # NOTE: If there exists a layer with remote content, then we consider this manifest
+    # to not support legacy images.
+    if self.has_remote_layer:
+      return None
+
     return list(self.layers_with_v1_ids)[-1].v1_id
 
   @property
   def legacy_image_ids(self):
+    if self.has_remote_layer:
+      return None
+
     return [l.v1_id for l in self.layers_with_v1_ids]
 
   @property
   def blob_digests(self):
     return [str(layer.digest) for layer in self.layers] + [str(self.config.digest)]
 
+  @property
+  def local_blob_digests(self):
+    return ([str(layer.digest) for layer in self.layers if not layer.urls] +
+            [str(self.config.digest)])
+
   def get_manifest_labels(self, lookup_config_fn):
     return self._get_built_config(lookup_config_fn).labels
 
@@ -218,6 +243,7 @@ class DockerSchema2Manifest(ManifestInterface):
 
   @property
   def layers_with_v1_ids(self):
+    assert not self.has_remote_layer
     digest_history = hashlib.sha256()
     v1_layer_parent_id = None
     v1_layer_id = None
@@ -240,6 +266,7 @@ class DockerSchema2Manifest(ManifestInterface):
         this schema. The `lookup_config_fn` is a function that, when given the config
         digest SHA, returns the associated configuration JSON bytes for this schema.
     """
+    assert not self.has_remote_layer
     schema2_config = self._get_built_config(lookup_config_fn)
 
     # Build the V1 IDs for the layers.
@@ -253,6 +280,8 @@ class DockerSchema2Manifest(ManifestInterface):
     return v1_builder
 
   def generate_legacy_layers(self, images_map, lookup_config_fn):
+    assert not self.has_remote_layer
+
     # NOTE: We use the DockerSchema1ManifestBuilder here because it already contains
     # the logic for generating the DockerV1Metadata. All of this will go away once we get
     # rid of legacy images in the database, so this is a temporary solution.
@@ -261,6 +290,9 @@ class DockerSchema2Manifest(ManifestInterface):
     return v1_builder.build().generate_legacy_layers(images_map, lookup_config_fn)
 
   def get_v1_compatible_manifest(self, namespace_name, repo_name, tag_name, lookup_fn):
+    if self.has_remote_layer:
+      return None
+
     v1_builder = DockerSchema1ManifestBuilder(namespace_name, repo_name, tag_name)
     self.populate_schema1_builder(v1_builder, lookup_fn)
     return v1_builder.build()

image/docker/schema2/test/test_list.py

@@ -28,7 +28,7 @@ MANIFESTLIST_BYTES = json.dumps({
   "manifests": [
     {
       "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-      "size": 983,
+      "size": 946,
       "digest": "sha256:e6",
       "platform": {
         "architecture": "ppc64le",
@@ -56,7 +56,7 @@ NO_AMD_MANIFESTLIST_BYTES = json.dumps({
   "manifests": [
     {
       "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-      "size": 983,
+      "size": 946,
       "digest": "sha256:e6",
       "platform": {
         "architecture": "ppc64le",
@@ -76,7 +76,7 @@ def test_valid_manifestlist():
   manifestlist = DockerSchema2ManifestList(MANIFESTLIST_BYTES)
   assert len(manifestlist.manifests(_get_manifest)) == 2
   assert (manifestlist.digest ==
-          'sha256:7e22fdbe49736329786c9b4fdc154cc9251b190ca6b4cf33aed00efc0fc3df25')
+          'sha256:340d7dadea77035533a2d43e8ff711ecca1965978a6e7699b87e32b35f76038d')
 
   assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
   assert manifestlist.bytes == MANIFESTLIST_BYTES
@@ -109,7 +109,7 @@ def test_get_v1_compatible_manifest_no_matching_list():
   manifestlist = DockerSchema2ManifestList(NO_AMD_MANIFESTLIST_BYTES)
   assert len(manifestlist.manifests(_get_manifest)) == 1
   assert (manifestlist.digest ==
-          'sha256:50150251101420a020ab4a3e77e9d167a18b09bd4eeb0cc65e0eafab95cf79cf')
+          'sha256:40ed1cfe692333bfa519a9bfed9676975a990fff5afd35efa628320c39c793ca')
 
   assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
   assert manifestlist.bytes == NO_AMD_MANIFESTLIST_BYTES

image/docker/schema2/test/test_manifest.py

@@ -31,6 +31,38 @@ MANIFEST_BYTES = json.dumps({
     "size": 1885,
     "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
   },
+  "layers": [
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 1234,
+      "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736",
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 32654,
+      "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 16724,
+      "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 73109,
+      "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736"
+    },
+  ],
+})
+
+REMOTE_MANIFEST_BYTES = json.dumps({
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+  "config": {
+    "mediaType": "application/vnd.docker.container.image.v1+json",
+    "size": 1885,
+    "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
+  },
   "layers": [
     {
       "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
@@ -61,12 +93,12 @@ def test_valid_manifest():
   assert manifest.config.size == 1885
   assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
   assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
+  assert not manifest.has_remote_layer
 
   assert len(manifest.layers) == 4
-  assert manifest.layers[0].is_remote
   assert manifest.layers[0].compressed_size == 1234
   assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
-  assert manifest.layers[0].urls
+  assert not manifest.layers[0].is_remote
 
   assert manifest.leaf_layer == manifest.layers[3]
   assert not manifest.leaf_layer.is_remote
@@ -75,6 +107,37 @@ def test_valid_manifest():
   blob_digests = list(manifest.blob_digests)
   expected = [str(layer.digest) for layer in manifest.layers] + [manifest.config.digest]
   assert blob_digests == expected
+  assert list(manifest.local_blob_digests) == expected
+
+
+def test_valid_remote_manifest():
+  manifest = DockerSchema2Manifest(REMOTE_MANIFEST_BYTES)
+  assert manifest.config.size == 1885
+  assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
+  assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
+  assert manifest.has_remote_layer
+
+  assert len(manifest.layers) == 4
+  assert manifest.layers[0].compressed_size == 1234
+  assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
+  assert manifest.layers[0].is_remote
+  assert manifest.layers[0].urls == ['http://some/url']
+
+  assert manifest.leaf_layer == manifest.layers[3]
+  assert not manifest.leaf_layer.is_remote
+  assert manifest.leaf_layer.compressed_size == 73109
+
+  expected = set([str(layer.digest) for layer in manifest.layers] + [manifest.config.digest])
+
+  blob_digests = set(manifest.blob_digests)
+  local_digests = set(manifest.local_blob_digests)
+
+  assert blob_digests == expected
+  assert local_digests == (expected - {manifest.layers[0].digest})
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.legacy_image_ids is None
 
 
 def test_schema2_builder():
@@ -110,6 +173,7 @@ def test_get_manifest_labels():
 
 def test_build_schema1():
   manifest = DockerSchema2Manifest(MANIFEST_BYTES)
+  assert not manifest.has_remote_layer
 
   builder = DockerSchema1ManifestBuilder('somenamespace', 'somename', 'sometag')
   manifest.populate_schema1_builder(builder, lambda digest: CONFIG_BYTES)
@@ -209,3 +273,22 @@ def test_generate_legacy_layers():
   assert legacy_layers[0].parent_image_id is None
 
   assert legacy_layers[0].image_id != legacy_layers[1]
+
+
+def test_remote_layer_manifest():
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest('sha256:abcd', 1234)
+  builder.add_layer('sha256:adef', 1234, urls=['http://some/url'])
+  builder.add_layer('sha256:1352', 4567)
+  builder.add_layer('sha256:1353', 4567)
+  manifest = builder.build()
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.legacy_image_ids is None
+
+  schema1 = manifest.get_v1_compatible_manifest('somenamespace', 'somename', 'sometag', None)
+  assert schema1 is None
+
+  assert set(manifest.blob_digests) == {'sha256:adef', 'sha256:abcd', 'sha256:1352', 'sha256:1353'}
+  assert set(manifest.local_blob_digests) == {'sha256:abcd', 'sha256:1352', 'sha256:1353'}