Add support for pushing and pulling schema 2 manifests with remote layers

This is required for windows image support

data/model/oci/manifest.py

@@ -57,7 +57,7 @@ def get_or_create_manifest(repository_id, manifest_interface_instance, storage):
 
 
 def _create_manifest(repository_id, manifest_interface_instance, storage):
-  digests = set(manifest_interface_instance.blob_digests)
+  digests = set(manifest_interface_instance.local_blob_digests)
 
   def _lookup_digest(digest):
     return _retrieve_bytes_in_storage(repository_id, digest, storage)
@@ -111,13 +111,15 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
       child_manifest_label_dicts.append(labels)
 
   # Ensure all the blobs in the manifest exist.
-  query = lookup_repo_storages_by_content_checksum(repository_id, digests)
-  blob_map = {s.content_checksum: s for s in query}
-  for digest_str in digests:
-    if digest_str not in blob_map:
-      logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
-                     manifest_interface_instance.digest, repository_id)
-      return None
+  blob_map = {}
+  if digests:
+    query = lookup_repo_storages_by_content_checksum(repository_id, digests)
+    blob_map = {s.content_checksum: s for s in query}
+    for digest_str in digests:
+      if digest_str not in blob_map:
+        logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
+                       manifest_interface_instance.digest, repository_id)
+        return None
 
   # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
   # image.

data/model/oci/test/test_oci_manifest.py

@@ -239,3 +239,68 @@ def test_get_or_create_manifest_list(initialized_db):
 
   assert child_manifests[v1_manifest.digest].media_type.name == v1_manifest.media_type
   assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type
+
+
+def test_get_or_create_manifest_with_remote_layers(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  remote_digest = sha256_digest('something')
+
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(remote_digest, 1234, urls=['http://hello/world'])
+  builder.add_layer(random_digest, len(random_data))
+  manifest = builder.build()
+
+  assert remote_digest in manifest.blob_digests
+  assert remote_digest not in manifest.local_blob_digests
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.get_v1_compatible_manifest('foo', 'bar', 'baz', None) is None
+
+  # Write the manifest.
+  created_tuple = get_or_create_manifest(repository, manifest, storage)
+  assert created_tuple is not None
+
+  created_manifest = created_tuple.manifest
+  assert created_manifest
+  assert created_manifest.media_type.name == manifest.media_type
+  assert created_manifest.digest == manifest.digest
+
+  # Verify the legacy image.
+  legacy_image = get_legacy_image_for_manifest(created_manifest)
+  assert legacy_image is None
+
+  # Verify the linked blobs.
+  blob_digests = {mb.blob.content_checksum for mb
+                  in ManifestBlob.select().where(ManifestBlob.manifest == created_manifest)}
+
+  assert random_digest in blob_digests
+  assert config_digest in blob_digests
+  assert remote_digest not in blob_digests

data/model/tag.py

@@ -766,7 +766,7 @@ def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_laye
     raise DataModelException('Invalid image with id: %s' % leaf_layer_id)
 
   storage_ids = set()
-  for blob_digest in manifest.blob_digests:
+  for blob_digest in manifest.local_blob_digests:
     image_storage_id = storage_id_map.get(blob_digest)
     if image_storage_id is None:
       logger.error('Missing blob for manifest `%s` in: %s', blob_digest, storage_id_map)

data/registry_model/datatypes.py

@@ -153,6 +153,14 @@ class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_sta
     """
     return legacy_image
 
+  @property
+  @optionalinput('legacy_image')
+  def legacy_image_if_present(self, legacy_image):
+    """ Returns the legacy Docker V1-style image for this tag. Note that this
+        will be None for tags whose manifests point to other manifests instead of images.
+    """
+    return legacy_image
+
   @property
   def id(self):
     """ The ID of this tag for pagination purposes only. """
@@ -266,7 +274,8 @@ class SecurityScanStatus(Enum):
 class ManifestLayer(namedtuple('ManifestLayer', ['layer_info', 'blob'])):
   """ Represents a single layer in a manifest. The `layer_info` data will be manifest-type specific,
       but will have a few expected fields (such as `digest`). The `blob` represents the associated
-      blob for this layer, optionally with placements.
+      blob for this layer, optionally with placements. If the layer is a remote layer, the blob will
+      be None.
   """
 
   def estimated_size(self, estimate_multiplier):

data/registry_model/shared.py

@@ -317,12 +317,18 @@ class SharedModel:
       logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
       return None
 
-    blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
-                                                                        parsed.blob_digests)
-    storage_map = {blob.content_checksum: blob for blob in blob_query}
+    storage_map = {}
+    if parsed.local_blob_digests:
+      blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
+                                                                          parsed.local_blob_digests)
+      storage_map = {blob.content_checksum: blob for blob in blob_query}
 
     manifest_layers = []
     for layer in parsed.layers:
+      if layer.is_remote:
+        manifest_layers.append(ManifestLayer(layer, None))
+        continue
+
       digest_str = str(layer.digest)
       if digest_str not in storage_map:
         logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)

data/registry_model/test/test_interface.py

@@ -3,6 +3,7 @@ import json
 import uuid
 
 from datetime import datetime, timedelta
+from io import BytesIO
 
 import pytest
 
@@ -19,7 +20,9 @@ from data.cache.impl import InMemoryDataModelCache
 from data.registry_model.registry_pre_oci_model import PreOCIModel
 from data.registry_model.registry_oci_model import OCIModel
 from data.registry_model.datatypes import RepositoryReference
+from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
 from image.docker.schema1 import DockerSchema1ManifestBuilder
+from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
 
 from test.fixtures import *
 
@@ -32,6 +35,10 @@ def registry_model(request, initialized_db):
 def pre_oci_model(initialized_db):
   return PreOCIModel()
 
+@pytest.fixture()
+def oci_model(initialized_db):
+  return OCIModel()
+
 
 @pytest.mark.parametrize('names, expected', [
   (['unknown'], None),
@@ -481,6 +488,45 @@ def test_list_manifest_layers(repo_namespace, repo_name, registry_model):
       assert manifest_layer.estimated_size(1) is not None
 
 
+def test_manifest_remote_layers(oci_model):
+  # Create a config blob for testing.
+  config_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  app_config = {'TESTING': True}
+  repository_ref = oci_model.lookup_repository('devtable', 'simple')
+  with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500, 500)) as upload:
+    upload.upload_chunk(app_config, BytesIO(config_json))
+    blob = upload.commit_to_blob(app_config)
+
+  # Create the manifest in the repo.
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(blob.digest, blob.compressed_size)
+  builder.add_layer('sha256:abcd', 1234, urls=['http://hello/world'])
+  manifest = builder.build()
+
+  created_manifest, _ = oci_model.create_manifest_and_retarget_tag(repository_ref, manifest,
+                                                                   'sometag', storage)
+  assert created_manifest
+
+  layers = oci_model.list_manifest_layers(created_manifest)
+  assert len(layers) == 1
+  assert layers[0].layer_info.is_remote
+  assert layers[0].layer_info.urls == ['http://hello/world']
+  assert layers[0].blob is None
+
+
 def test_derived_image(registry_model):
   # Clear all existing derived storage.
   DerivedStorageForImage.delete().execute()