Add support for pushing and pulling schema 2 manifests with remote layers

This is required for windows image support

data/model/oci/manifest.py

@@ -57,7 +57,7 @@ def get_or_create_manifest(repository_id, manifest_interface_instance, storage):
 
 
 def _create_manifest(repository_id, manifest_interface_instance, storage):
-  digests = set(manifest_interface_instance.blob_digests)
+  digests = set(manifest_interface_instance.local_blob_digests)
 
   def _lookup_digest(digest):
     return _retrieve_bytes_in_storage(repository_id, digest, storage)
@@ -111,13 +111,15 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
       child_manifest_label_dicts.append(labels)
 
   # Ensure all the blobs in the manifest exist.
-  query = lookup_repo_storages_by_content_checksum(repository_id, digests)
+  blob_map = {}
-  blob_map = {s.content_checksum: s for s in query}
+  if digests:
-  for digest_str in digests:
+    query = lookup_repo_storages_by_content_checksum(repository_id, digests)
-    if digest_str not in blob_map:
+    blob_map = {s.content_checksum: s for s in query}
-      logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
+    for digest_str in digests:
-                     manifest_interface_instance.digest, repository_id)
+      if digest_str not in blob_map:
-      return None
+        logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
+                       manifest_interface_instance.digest, repository_id)
+        return None
 
   # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
   # image.

data/model/oci/test/test_oci_manifest.py

@@ -239,3 +239,68 @@ def test_get_or_create_manifest_list(initialized_db):
 
   assert child_manifests[v1_manifest.digest].media_type.name == v1_manifest.media_type
   assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type
+
+
+def test_get_or_create_manifest_with_remote_layers(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  remote_digest = sha256_digest('something')
+
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(remote_digest, 1234, urls=['http://hello/world'])
+  builder.add_layer(random_digest, len(random_data))
+  manifest = builder.build()
+
+  assert remote_digest in manifest.blob_digests
+  assert remote_digest not in manifest.local_blob_digests
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.get_v1_compatible_manifest('foo', 'bar', 'baz', None) is None
+
+  # Write the manifest.
+  created_tuple = get_or_create_manifest(repository, manifest, storage)
+  assert created_tuple is not None
+
+  created_manifest = created_tuple.manifest
+  assert created_manifest
+  assert created_manifest.media_type.name == manifest.media_type
+  assert created_manifest.digest == manifest.digest
+
+  # Verify the legacy image.
+  legacy_image = get_legacy_image_for_manifest(created_manifest)
+  assert legacy_image is None
+
+  # Verify the linked blobs.
+  blob_digests = {mb.blob.content_checksum for mb
+                  in ManifestBlob.select().where(ManifestBlob.manifest == created_manifest)}
+
+  assert random_digest in blob_digests
+  assert config_digest in blob_digests
+  assert remote_digest not in blob_digests

data/model/tag.py

@@ -766,7 +766,7 @@ def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_laye
     raise DataModelException('Invalid image with id: %s' % leaf_layer_id)
 
   storage_ids = set()
-  for blob_digest in manifest.blob_digests:
+  for blob_digest in manifest.local_blob_digests:
     image_storage_id = storage_id_map.get(blob_digest)
     if image_storage_id is None:
       logger.error('Missing blob for manifest `%s` in: %s', blob_digest, storage_id_map)

data/registry_model/datatypes.py

@@ -153,6 +153,14 @@ class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_sta
     """
     return legacy_image
 
+  @property
+  @optionalinput('legacy_image')
+  def legacy_image_if_present(self, legacy_image):
+    """ Returns the legacy Docker V1-style image for this tag. Note that this
+        will be None for tags whose manifests point to other manifests instead of images.
+    """
+    return legacy_image
+
   @property
   def id(self):
     """ The ID of this tag for pagination purposes only. """
@@ -266,7 +274,8 @@ class SecurityScanStatus(Enum):
 class ManifestLayer(namedtuple('ManifestLayer', ['layer_info', 'blob'])):
   """ Represents a single layer in a manifest. The `layer_info` data will be manifest-type specific,
       but will have a few expected fields (such as `digest`). The `blob` represents the associated
-      blob for this layer, optionally with placements.
+      blob for this layer, optionally with placements. If the layer is a remote layer, the blob will
+      be None.
   """
 
   def estimated_size(self, estimate_multiplier):

data/registry_model/shared.py

@@ -317,12 +317,18 @@ class SharedModel:
       logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
       return None
 
-    blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
+    storage_map = {}
-                                                                        parsed.blob_digests)
+    if parsed.local_blob_digests:
-    storage_map = {blob.content_checksum: blob for blob in blob_query}
+      blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
+                                                                          parsed.local_blob_digests)
+      storage_map = {blob.content_checksum: blob for blob in blob_query}
 
     manifest_layers = []
     for layer in parsed.layers:
+      if layer.is_remote:
+        manifest_layers.append(ManifestLayer(layer, None))
+        continue
+
       digest_str = str(layer.digest)
       if digest_str not in storage_map:
         logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)

data/registry_model/test/test_interface.py

@@ -3,6 +3,7 @@ import json
 import uuid
 
 from datetime import datetime, timedelta
+from io import BytesIO
 
 import pytest
 
@@ -19,7 +20,9 @@ from data.cache.impl import InMemoryDataModelCache
 from data.registry_model.registry_pre_oci_model import PreOCIModel
 from data.registry_model.registry_oci_model import OCIModel
 from data.registry_model.datatypes import RepositoryReference
+from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
 from image.docker.schema1 import DockerSchema1ManifestBuilder
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
 
 from test.fixtures import *
 
@@ -32,6 +35,10 @@ def registry_model(request, initialized_db):
 def pre_oci_model(initialized_db):
   return PreOCIModel()
 
+@pytest.fixture()
+def oci_model(initialized_db):
+  return OCIModel()
+
 
 @pytest.mark.parametrize('names, expected', [
   (['unknown'], None),
@@ -481,6 +488,45 @@ def test_list_manifest_layers(repo_namespace, repo_name, registry_model):
       assert manifest_layer.estimated_size(1) is not None
 
 
+def test_manifest_remote_layers(oci_model):
+  # Create a config blob for testing.
+  config_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  app_config = {'TESTING': True}
+  repository_ref = oci_model.lookup_repository('devtable', 'simple')
+  with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500, 500)) as upload:
+    upload.upload_chunk(app_config, BytesIO(config_json))
+    blob = upload.commit_to_blob(app_config)
+
+  # Create the manifest in the repo.
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(blob.digest, blob.compressed_size)
+  builder.add_layer('sha256:abcd', 1234, urls=['http://hello/world'])
+  manifest = builder.build()
+
+  created_manifest, _ = oci_model.create_manifest_and_retarget_tag(repository_ref, manifest,
+                                                                   'sometag', storage)
+  assert created_manifest
+
+  layers = oci_model.list_manifest_layers(created_manifest)
+  assert len(layers) == 1
+  assert layers[0].layer_info.is_remote
+  assert layers[0].layer_info.urls == ['http://hello/world']
+  assert layers[0].blob is None
+
+
 def test_derived_image(registry_model):
   # Clear all existing derived storage.
   DerivedStorageForImage.delete().execute()

endpoints/v1/tag.py

@@ -30,7 +30,8 @@ def get_tags(namespace_name, repo_name):
     # TODO(jschorr): Change this to normalize manifest lists back to their legacy image
     # (if applicable).
     tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
-    tag_map = {tag.name: tag.legacy_image.docker_image_id for tag in tags if tag.legacy_image}
+    tag_map = {tag.name: tag.legacy_image.docker_image_id
+               for tag in tags if tag.legacy_image_if_present}
     return jsonify(tag_map)
 
   abort(403)

image/docker/interfaces.py

@@ -53,6 +53,13 @@ class ManifestInterface(object):
         config as a blob, the blob will be included here.
     """
 
+  @abstractproperty
+  def local_blob_digests(self):
+    """ Returns an iterator over all the *non-remote* blob digests referenced by this manifest,
+        from base to leaf. The blob digests are strings with prefixes. For manifests that reference
+        config as a blob, the blob will be included here.
+    """
+
   @abstractmethod
   def child_manifests(self, lookup_manifest_fn):
     """ Returns an iterator of all manifests that live under this manifest, if any or None if not

image/docker/schema1.py

@@ -73,7 +73,7 @@ class InvalidSchema1Signature(ManifestException):
 
 
 class Schema1Layer(namedtuple('Schema1Layer', ['digest', 'v1_metadata', 'raw_v1_metadata',
-                                               'compressed_size'])):
+                                               'compressed_size', 'is_remote'])):
   """
   Represents all of the data about an individual layer in a given Manifest.
   This is the union of the fsLayers (digest) and the history entries (v1_compatibility).
@@ -302,6 +302,10 @@ class DockerSchema1Manifest(ManifestInterface):
   def blob_digests(self):
     return [str(layer.digest) for layer in self.layers]
 
+  @property
+  def local_blob_digests(self):
+    return self.blob_digests
+
   def child_manifests(self, lookup_manifest_fn):
     return None
 
@@ -349,7 +353,7 @@ class DockerSchema1Manifest(ManifestInterface):
                                     command, labels)
 
       compressed_size = v1_metadata.get('Size')
-      yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size)
+      yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size, False)
 
   @property
   def _payload(self):

image/docker/schema2/list.py

@@ -224,6 +224,10 @@ class DockerSchema2ManifestList(ManifestInterface):
     manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]
     return [m[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY] for m in manifests]
 
+  @property
+  def local_blob_digests(self):
+    return self.blob_digests
+
   @lru_cache(maxsize=1)
   def manifests(self, lookup_manifest_fn):
     """ Returns the manifests in the list. The `lookup_manifest_fn` is a function

image/docker/schema2/manifest.py

@@ -136,6 +136,10 @@ class DockerSchema2Manifest(ManifestInterface):
     except ValidationError as ve:
       raise MalformedSchema2Manifest('manifest data does not match schema: %s' % ve)
 
+    for layer in self.layers:
+      if layer.is_remote and not layer.urls:
+        raise MalformedSchema2Manifest('missing `urls` for remote layer')
+
   @property
   def schema_version(self):
     return 2
@@ -169,18 +173,39 @@ class DockerSchema2Manifest(ManifestInterface):
   def leaf_layer(self):
     return self.layers[-1]
 
+  @property
+  def has_remote_layer(self):
+    for layer in self.layers:
+      if layer.is_remote:
+        return True
+
+    return False
+
   @property
   def leaf_layer_v1_image_id(self):
+    # NOTE: If there exists a layer with remote content, then we consider this manifest
+    # to not support legacy images.
+    if self.has_remote_layer:
+      return None
+
     return list(self.layers_with_v1_ids)[-1].v1_id
 
   @property
   def legacy_image_ids(self):
+    if self.has_remote_layer:
+      return None
+
     return [l.v1_id for l in self.layers_with_v1_ids]
 
   @property
   def blob_digests(self):
     return [str(layer.digest) for layer in self.layers] + [str(self.config.digest)]
 
+  @property
+  def local_blob_digests(self):
+    return ([str(layer.digest) for layer in self.layers if not layer.urls] +
+            [str(self.config.digest)])
+
   def get_manifest_labels(self, lookup_config_fn):
     return self._get_built_config(lookup_config_fn).labels
 
@@ -218,6 +243,7 @@ class DockerSchema2Manifest(ManifestInterface):
 
   @property
   def layers_with_v1_ids(self):
+    assert not self.has_remote_layer
     digest_history = hashlib.sha256()
     v1_layer_parent_id = None
     v1_layer_id = None
@@ -240,6 +266,7 @@ class DockerSchema2Manifest(ManifestInterface):
         this schema. The `lookup_config_fn` is a function that, when given the config
         digest SHA, returns the associated configuration JSON bytes for this schema.
     """
+    assert not self.has_remote_layer
     schema2_config = self._get_built_config(lookup_config_fn)
 
     # Build the V1 IDs for the layers.
@@ -253,6 +280,8 @@ class DockerSchema2Manifest(ManifestInterface):
     return v1_builder
 
   def generate_legacy_layers(self, images_map, lookup_config_fn):
+    assert not self.has_remote_layer
+
     # NOTE: We use the DockerSchema1ManifestBuilder here because it already contains
     # the logic for generating the DockerV1Metadata. All of this will go away once we get
     # rid of legacy images in the database, so this is a temporary solution.
@@ -261,6 +290,9 @@ class DockerSchema2Manifest(ManifestInterface):
     return v1_builder.build().generate_legacy_layers(images_map, lookup_config_fn)
 
   def get_v1_compatible_manifest(self, namespace_name, repo_name, tag_name, lookup_fn):
+    if self.has_remote_layer:
+      return None
+
     v1_builder = DockerSchema1ManifestBuilder(namespace_name, repo_name, tag_name)
     self.populate_schema1_builder(v1_builder, lookup_fn)
     return v1_builder.build()

image/docker/schema2/test/test_list.py

@@ -28,7 +28,7 @@ MANIFESTLIST_BYTES = json.dumps({
   "manifests": [
     {
       "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-      "size": 983,
+      "size": 946,
       "digest": "sha256:e6",
       "platform": {
         "architecture": "ppc64le",
@@ -56,7 +56,7 @@ NO_AMD_MANIFESTLIST_BYTES = json.dumps({
   "manifests": [
     {
       "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-      "size": 983,
+      "size": 946,
       "digest": "sha256:e6",
       "platform": {
         "architecture": "ppc64le",
@@ -76,7 +76,7 @@ def test_valid_manifestlist():
   manifestlist = DockerSchema2ManifestList(MANIFESTLIST_BYTES)
   assert len(manifestlist.manifests(_get_manifest)) == 2
   assert (manifestlist.digest ==
-          'sha256:7e22fdbe49736329786c9b4fdc154cc9251b190ca6b4cf33aed00efc0fc3df25')
+          'sha256:340d7dadea77035533a2d43e8ff711ecca1965978a6e7699b87e32b35f76038d')
 
   assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
   assert manifestlist.bytes == MANIFESTLIST_BYTES
@@ -109,7 +109,7 @@ def test_get_v1_compatible_manifest_no_matching_list():
   manifestlist = DockerSchema2ManifestList(NO_AMD_MANIFESTLIST_BYTES)
   assert len(manifestlist.manifests(_get_manifest)) == 1
   assert (manifestlist.digest ==
-          'sha256:50150251101420a020ab4a3e77e9d167a18b09bd4eeb0cc65e0eafab95cf79cf')
+          'sha256:40ed1cfe692333bfa519a9bfed9676975a990fff5afd35efa628320c39c793ca')
 
   assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
   assert manifestlist.bytes == NO_AMD_MANIFESTLIST_BYTES

image/docker/schema2/test/test_manifest.py

@@ -31,6 +31,38 @@ MANIFEST_BYTES = json.dumps({
     "size": 1885,
     "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
   },
+  "layers": [
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 1234,
+      "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736",
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 32654,
+      "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 16724,
+      "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"
+    },
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 73109,
+      "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736"
+    },
+  ],
+})
+
+REMOTE_MANIFEST_BYTES = json.dumps({
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+  "config": {
+    "mediaType": "application/vnd.docker.container.image.v1+json",
+    "size": 1885,
+    "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
+  },
   "layers": [
     {
       "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
@@ -61,12 +93,12 @@ def test_valid_manifest():
   assert manifest.config.size == 1885
   assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
   assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
+  assert not manifest.has_remote_layer
 
   assert len(manifest.layers) == 4
-  assert manifest.layers[0].is_remote
   assert manifest.layers[0].compressed_size == 1234
   assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
-  assert manifest.layers[0].urls
+  assert not manifest.layers[0].is_remote
 
   assert manifest.leaf_layer == manifest.layers[3]
   assert not manifest.leaf_layer.is_remote
@@ -75,6 +107,37 @@ def test_valid_manifest():
   blob_digests = list(manifest.blob_digests)
   expected = [str(layer.digest) for layer in manifest.layers] + [manifest.config.digest]
   assert blob_digests == expected
+  assert list(manifest.local_blob_digests) == expected
+
+
+def test_valid_remote_manifest():
+  manifest = DockerSchema2Manifest(REMOTE_MANIFEST_BYTES)
+  assert manifest.config.size == 1885
+  assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
+  assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
+  assert manifest.has_remote_layer
+
+  assert len(manifest.layers) == 4
+  assert manifest.layers[0].compressed_size == 1234
+  assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
+  assert manifest.layers[0].is_remote
+  assert manifest.layers[0].urls == ['http://some/url']
+
+  assert manifest.leaf_layer == manifest.layers[3]
+  assert not manifest.leaf_layer.is_remote
+  assert manifest.leaf_layer.compressed_size == 73109
+
+  expected = set([str(layer.digest) for layer in manifest.layers] + [manifest.config.digest])
+
+  blob_digests = set(manifest.blob_digests)
+  local_digests = set(manifest.local_blob_digests)
+
+  assert blob_digests == expected
+  assert local_digests == (expected - {manifest.layers[0].digest})
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.legacy_image_ids is None
 
 
 def test_schema2_builder():
@@ -110,6 +173,7 @@ def test_get_manifest_labels():
 
 def test_build_schema1():
   manifest = DockerSchema2Manifest(MANIFEST_BYTES)
+  assert not manifest.has_remote_layer
 
   builder = DockerSchema1ManifestBuilder('somenamespace', 'somename', 'sometag')
   manifest.populate_schema1_builder(builder, lambda digest: CONFIG_BYTES)
@@ -209,3 +273,22 @@ def test_generate_legacy_layers():
   assert legacy_layers[0].parent_image_id is None
 
   assert legacy_layers[0].image_id != legacy_layers[1]
+
+
+def test_remote_layer_manifest():
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest('sha256:abcd', 1234)
+  builder.add_layer('sha256:adef', 1234, urls=['http://some/url'])
+  builder.add_layer('sha256:1352', 4567)
+  builder.add_layer('sha256:1353', 4567)
+  manifest = builder.build()
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.legacy_image_ids is None
+
+  schema1 = manifest.get_v1_compatible_manifest('somenamespace', 'somename', 'sometag', None)
+  assert schema1 is None
+
+  assert set(manifest.blob_digests) == {'sha256:adef', 'sha256:abcd', 'sha256:1352', 'sha256:1353'}
+  assert set(manifest.local_blob_digests) == {'sha256:abcd', 'sha256:1352', 'sha256:1353'}

test/registry/protocol_fixtures.py

@@ -92,6 +92,20 @@ def multi_layer_images():
   ]
 
 
+@pytest.fixture(scope="session")
+def remote_images():
+  """ Returns images with at least one remote layer for push and pull testing. """
+  # Note: order is from base layer down to leaf.
+  remote_bytes = layer_bytes_for_contents('remote contents')
+  parent_bytes = layer_bytes_for_contents('parent contents')
+  image_bytes = layer_bytes_for_contents('some contents')
+  return [
+    Image(id='remoteid', bytes=remote_bytes, parent_id=None, urls=['http://some/url']),
+    Image(id='parentid', bytes=parent_bytes, parent_id='remoteid'),
+    Image(id='someid', bytes=image_bytes, parent_id='parentid'),
+  ]
+
+
 @pytest.fixture(scope="session")
 def jwk():
   return RSAKey(key=RSA.generate(2048))

test/registry/protocol_v1.py

@@ -93,7 +93,6 @@ class V1Protocol(RegistryProtocol):
 
     # GET /v1/repositories/{namespace}/{repository}/tags
     image_ids = self.conduct(session, 'GET', prefix + 'tags', headers=headers).json()
-    assert len(image_ids.values()) >= len(tag_names)
 
     for tag_name in tag_names:
       if tag_name not in image_ids:
@@ -145,6 +144,8 @@ class V1Protocol(RegistryProtocol):
     headers['Authorization'] = 'token ' + result.headers['www-authenticate']
 
     for image in images:
+      assert image.urls is None
+
       # PUT /v1/images/{imageID}/json
       image_json_data = {'id': image.id}
       if image.size is not None:

test/registry/protocol_v2.py

@@ -209,13 +209,15 @@ class V2Protocol(RegistryProtocol):
     builder = DockerSchema2ManifestBuilder()
     for image in images:
       checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
-      blobs[checksum] = image.bytes
+
+      if image.urls is None:
+        blobs[checksum] = image.bytes
 
       # If invalid blob references were requested, just make it up.
       if options.manifest_invalid_blob_references:
         checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
 
-      builder.add_layer(checksum, len(image.bytes))
+      builder.add_layer(checksum, len(image.bytes), urls=image.urls)
 
     config = {
       "os": "linux",
@@ -245,6 +247,8 @@ class V2Protocol(RegistryProtocol):
     builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
 
     for image in reversed(images):
+      assert image.urls is None
+
       checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
       blobs[checksum] = image.bytes
 
@@ -498,12 +502,16 @@ class V2Protocol(RegistryProtocol):
 
       # Verify the layers.
       for index, layer in enumerate(manifest.layers):
+        # If the layer is remote, then we expect the blob to *not* exist in the system.
+        expected_status = 404 if images[index].urls else 200
         result = self.conduct(session, 'GET',
                               '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name),
                                                    layer.digest),
-                              expected_status=200,
+                              expected_status=expected_status,
                               headers=headers)
-        assert result.content == images[index].bytes
+        
+        if expected_status == 200:
+          assert result.content == images[index].bytes
 
     return PullResult(manifests=manifests, image_ids=image_ids)
 

test/registry/protocols.py

@@ -7,8 +7,8 @@ from cStringIO import StringIO
 from enum import Enum, unique
 from six import add_metaclass
 
-Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created'])
+Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created', 'urls'])
-Image.__new__.__defaults__ = (None, None, None)
+Image.__new__.__defaults__ = (None, None, None, None)
 
 PushResult = namedtuple('PushResult', ['manifests', 'headers'])
 PullResult = namedtuple('PullResult', ['manifests', 'image_ids'])

test/registry/registry_tests.py

@@ -21,6 +21,7 @@ from app import instance_keys
 from data.model.tag import list_repository_tags
 from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
 from image.docker.schema2.list import DockerSchema2ManifestListBuilder
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
 from util.security.registry_jwt import decode_bearer_header
 from util.timedeltastring import convert_to_timedelta
 
@@ -1450,3 +1451,26 @@ def test_push_pull_manifest_list(v22_protocol, basic_images, different_images, l
   # Pull and verify the manifest list.
   v22_protocol.pull_list(liveserver_session, 'devtable', 'newrepo', 'latest', manifestlist,
                          credentials=credentials, options=options)
+
+
+def test_push_pull_manifest_remote_layers(v22_protocol, legacy_puller, liveserver_session,
+                                          app_reloader, remote_images, data_model):
+  """ Test: Push a new tag with a manifest which contains at least one remote layer, and then
+      pull that manifest back.
+  """
+  if data_model != 'oci_model':
+    return
+
+  credentials = ('devtable', 'password')
+
+  # Push a new repository.
+  v22_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
+                    credentials=credentials)
+
+  # Pull the repository to verify.
+  v22_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
+                    credentials=credentials)
+
+  # Ensure that the image cannot be pulled by a legacy protocol.
+  legacy_puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
+                     credentials=credentials, expected_failure=Failures.UNKNOWN_TAG)

workers/manifestbackfillworker.py

@@ -63,6 +63,10 @@ class BrokenManifest(ManifestInterface):
   def blob_digests(self):
     return []
   
+  @property
+  def local_blob_digests(self):
+    return []
+
   def child_manifests(self, lookup_manifest_fn):
     return None