Add support for pushing and pulling schema 2 manifests with remote layers

This is required for windows image support

data/model/oci/manifest.py

@@ -57,7 +57,7 @@ def get_or_create_manifest(repository_id, manifest_interface_instance, storage):
 
 
 def _create_manifest(repository_id, manifest_interface_instance, storage):
-  digests = set(manifest_interface_instance.blob_digests)
+  digests = set(manifest_interface_instance.local_blob_digests)
 
   def _lookup_digest(digest):
     return _retrieve_bytes_in_storage(repository_id, digest, storage)
@@ -111,13 +111,15 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
       child_manifest_label_dicts.append(labels)
 
   # Ensure all the blobs in the manifest exist.
-  query = lookup_repo_storages_by_content_checksum(repository_id, digests)
-  blob_map = {s.content_checksum: s for s in query}
-  for digest_str in digests:
-    if digest_str not in blob_map:
-      logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
-                     manifest_interface_instance.digest, repository_id)
-      return None
+  blob_map = {}
+  if digests:
+    query = lookup_repo_storages_by_content_checksum(repository_id, digests)
+    blob_map = {s.content_checksum: s for s in query}
+    for digest_str in digests:
+      if digest_str not in blob_map:
+        logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
+                       manifest_interface_instance.digest, repository_id)
+        return None
 
   # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
   # image.

data/model/oci/test/test_oci_manifest.py

@@ -239,3 +239,68 @@ def test_get_or_create_manifest_list(initialized_db):
 
   assert child_manifests[v1_manifest.digest].media_type.name == v1_manifest.media_type
   assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type
+
+
+def test_get_or_create_manifest_with_remote_layers(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+      {
+        "created": "2018-04-03T18:37:09.284840891Z",
+        "created_by": "do something",
+      },
+    ],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  # Add a blob of random data.
+  random_data = 'hello world'
+  _, random_digest = _populate_blob(random_data)
+
+  remote_digest = sha256_digest('something')
+
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(remote_digest, 1234, urls=['http://hello/world'])
+  builder.add_layer(random_digest, len(random_data))
+  manifest = builder.build()
+
+  assert remote_digest in manifest.blob_digests
+  assert remote_digest not in manifest.local_blob_digests
+
+  assert manifest.has_remote_layer
+  assert manifest.leaf_layer_v1_image_id is None
+  assert manifest.get_v1_compatible_manifest('foo', 'bar', 'baz', None) is None
+
+  # Write the manifest.
+  created_tuple = get_or_create_manifest(repository, manifest, storage)
+  assert created_tuple is not None
+
+  created_manifest = created_tuple.manifest
+  assert created_manifest
+  assert created_manifest.media_type.name == manifest.media_type
+  assert created_manifest.digest == manifest.digest
+
+  # Verify the legacy image.
+  legacy_image = get_legacy_image_for_manifest(created_manifest)
+  assert legacy_image is None
+
+  # Verify the linked blobs.
+  blob_digests = {mb.blob.content_checksum for mb
+                  in ManifestBlob.select().where(ManifestBlob.manifest == created_manifest)}
+
+  assert random_digest in blob_digests
+  assert config_digest in blob_digests
+  assert remote_digest not in blob_digests

data/model/tag.py

@@ -766,7 +766,7 @@ def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_laye
     raise DataModelException('Invalid image with id: %s' % leaf_layer_id)
 
   storage_ids = set()
-  for blob_digest in manifest.blob_digests:
+  for blob_digest in manifest.local_blob_digests:
     image_storage_id = storage_id_map.get(blob_digest)
     if image_storage_id is None:
       logger.error('Missing blob for manifest `%s` in: %s', blob_digest, storage_id_map)