Add support for pushing and pulling schema 2 manifests with remote layers

This is required for windows image support

data/registry_model/datatypes.py

@@ -153,6 +153,14 @@ class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_sta
     """
     return legacy_image
 
+  @property
+  @optionalinput('legacy_image')
+  def legacy_image_if_present(self, legacy_image):
+    """ Returns the legacy Docker V1-style image for this tag. Note that this
+        will be None for tags whose manifests point to other manifests instead of images.
+    """
+    return legacy_image
+
   @property
   def id(self):
     """ The ID of this tag for pagination purposes only. """
@@ -266,7 +274,8 @@ class SecurityScanStatus(Enum):
 class ManifestLayer(namedtuple('ManifestLayer', ['layer_info', 'blob'])):
   """ Represents a single layer in a manifest. The `layer_info` data will be manifest-type specific,
       but will have a few expected fields (such as `digest`). The `blob` represents the associated
-      blob for this layer, optionally with placements.
+      blob for this layer, optionally with placements. If the layer is a remote layer, the blob will
+      be None.
   """
 
   def estimated_size(self, estimate_multiplier):

data/registry_model/shared.py

@@ -317,12 +317,18 @@ class SharedModel:
       logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
       return None
 
-    blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
-                                                                        parsed.blob_digests)
-    storage_map = {blob.content_checksum: blob for blob in blob_query}
+    storage_map = {}
+    if parsed.local_blob_digests:
+      blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
+                                                                          parsed.local_blob_digests)
+      storage_map = {blob.content_checksum: blob for blob in blob_query}
 
     manifest_layers = []
     for layer in parsed.layers:
+      if layer.is_remote:
+        manifest_layers.append(ManifestLayer(layer, None))
+        continue
+
       digest_str = str(layer.digest)
       if digest_str not in storage_map:
         logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)

data/registry_model/test/test_interface.py

@@ -3,6 +3,7 @@ import json
 import uuid
 
 from datetime import datetime, timedelta
+from io import BytesIO
 
 import pytest
 
@@ -19,7 +20,9 @@ from data.cache.impl import InMemoryDataModelCache
 from data.registry_model.registry_pre_oci_model import PreOCIModel
 from data.registry_model.registry_oci_model import OCIModel
 from data.registry_model.datatypes import RepositoryReference
+from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
 from image.docker.schema1 import DockerSchema1ManifestBuilder
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
 
 from test.fixtures import *
 
@@ -32,6 +35,10 @@ def registry_model(request, initialized_db):
 def pre_oci_model(initialized_db):
   return PreOCIModel()
 
+@pytest.fixture()
+def oci_model(initialized_db):
+  return OCIModel()
+
 
 @pytest.mark.parametrize('names, expected', [
   (['unknown'], None),
@@ -481,6 +488,45 @@ def test_list_manifest_layers(repo_namespace, repo_name, registry_model):
       assert manifest_layer.estimated_size(1) is not None
 
 
+def test_manifest_remote_layers(oci_model):
+  # Create a config blob for testing.
+  config_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  app_config = {'TESTING': True}
+  repository_ref = oci_model.lookup_repository('devtable', 'simple')
+  with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500, 500)) as upload:
+    upload.upload_chunk(app_config, BytesIO(config_json))
+    blob = upload.commit_to_blob(app_config)
+
+  # Create the manifest in the repo.
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(blob.digest, blob.compressed_size)
+  builder.add_layer('sha256:abcd', 1234, urls=['http://hello/world'])
+  manifest = builder.build()
+
+  created_manifest, _ = oci_model.create_manifest_and_retarget_tag(repository_ref, manifest,
+                                                                   'sometag', storage)
+  assert created_manifest
+
+  layers = oci_model.list_manifest_layers(created_manifest)
+  assert len(layers) == 1
+  assert layers[0].layer_info.is_remote
+  assert layers[0].layer_info.urls == ['http://hello/world']
+  assert layers[0].blob is None
+
+
 def test_derived_image(registry_model):
   # Clear all existing derived storage.
   DerivedStorageForImage.delete().execute()