Add support for pushing and pulling schema 2 manifests with remote layers

This is required for windows image support

test/registry/protocol_fixtures.py

@@ -92,6 +92,20 @@ def multi_layer_images():
   ]
 
 
+@pytest.fixture(scope="session")
+def remote_images():
+  """ Returns images with at least one remote layer for push and pull testing. """
+  # Note: order is from base layer down to leaf.
+  remote_bytes = layer_bytes_for_contents('remote contents')
+  parent_bytes = layer_bytes_for_contents('parent contents')
+  image_bytes = layer_bytes_for_contents('some contents')
+  return [
+    Image(id='remoteid', bytes=remote_bytes, parent_id=None, urls=['http://some/url']),
+    Image(id='parentid', bytes=parent_bytes, parent_id='remoteid'),
+    Image(id='someid', bytes=image_bytes, parent_id='parentid'),
+  ]
+
+
 @pytest.fixture(scope="session")
 def jwk():
   return RSAKey(key=RSA.generate(2048))

test/registry/protocol_v1.py

@@ -93,7 +93,6 @@ class V1Protocol(RegistryProtocol):
 
     # GET /v1/repositories/{namespace}/{repository}/tags
     image_ids = self.conduct(session, 'GET', prefix + 'tags', headers=headers).json()
-    assert len(image_ids.values()) >= len(tag_names)
 
     for tag_name in tag_names:
       if tag_name not in image_ids:
@@ -145,6 +144,8 @@ class V1Protocol(RegistryProtocol):
     headers['Authorization'] = 'token ' + result.headers['www-authenticate']
 
     for image in images:
+      assert image.urls is None
+
       # PUT /v1/images/{imageID}/json
       image_json_data = {'id': image.id}
       if image.size is not None:

test/registry/protocol_v2.py

@@ -209,13 +209,15 @@ class V2Protocol(RegistryProtocol):
     builder = DockerSchema2ManifestBuilder()
     for image in images:
       checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
-      blobs[checksum] = image.bytes
+
+      if image.urls is None:
+        blobs[checksum] = image.bytes
 
       # If invalid blob references were requested, just make it up.
       if options.manifest_invalid_blob_references:
         checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
 
-      builder.add_layer(checksum, len(image.bytes))
+      builder.add_layer(checksum, len(image.bytes), urls=image.urls)
 
     config = {
       "os": "linux",
@@ -245,6 +247,8 @@ class V2Protocol(RegistryProtocol):
     builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
 
     for image in reversed(images):
+      assert image.urls is None
+
       checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
       blobs[checksum] = image.bytes
 
@@ -498,12 +502,16 @@ class V2Protocol(RegistryProtocol):
 
       # Verify the layers.
       for index, layer in enumerate(manifest.layers):
+        # If the layer is remote, then we expect the blob to *not* exist in the system.
+        expected_status = 404 if images[index].urls else 200
         result = self.conduct(session, 'GET',
                               '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name),
                                                    layer.digest),
-                              expected_status=200,
+                              expected_status=expected_status,
                               headers=headers)
-        assert result.content == images[index].bytes
+        
+        if expected_status == 200:
+          assert result.content == images[index].bytes
 
     return PullResult(manifests=manifests, image_ids=image_ids)
 

test/registry/protocols.py

@@ -7,8 +7,8 @@ from cStringIO import StringIO
 from enum import Enum, unique
 from six import add_metaclass
 
-Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created'])
-Image.__new__.__defaults__ = (None, None, None)
+Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created', 'urls'])
+Image.__new__.__defaults__ = (None, None, None, None)
 
 PushResult = namedtuple('PushResult', ['manifests', 'headers'])
 PullResult = namedtuple('PullResult', ['manifests', 'image_ids'])

test/registry/registry_tests.py

@@ -21,6 +21,7 @@ from app import instance_keys
 from data.model.tag import list_repository_tags
 from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
 from image.docker.schema2.list import DockerSchema2ManifestListBuilder
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
 from util.security.registry_jwt import decode_bearer_header
 from util.timedeltastring import convert_to_timedelta
 
@@ -1450,3 +1451,26 @@ def test_push_pull_manifest_list(v22_protocol, basic_images, different_images, l
   # Pull and verify the manifest list.
   v22_protocol.pull_list(liveserver_session, 'devtable', 'newrepo', 'latest', manifestlist,
                          credentials=credentials, options=options)
+
+
+def test_push_pull_manifest_remote_layers(v22_protocol, legacy_puller, liveserver_session,
+                                          app_reloader, remote_images, data_model):
+  """ Test: Push a new tag with a manifest which contains at least one remote layer, and then
+      pull that manifest back.
+  """
+  if data_model != 'oci_model':
+    return
+
+  credentials = ('devtable', 'password')
+
+  # Push a new repository.
+  v22_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
+                    credentials=credentials)
+
+  # Pull the repository to verify.
+  v22_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
+                    credentials=credentials)
+
+  # Ensure that the image cannot be pulled by a legacy protocol.
+  legacy_puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
+                     credentials=credentials, expected_failure=Failures.UNKNOWN_TAG)