Add schema2 list support

image/docker/schema1.py

@@ -195,6 +195,10 @@ class DockerSchema1Manifest(object):
       if not verified:
         raise InvalidSchema1Signature()
 
+  @property
+  def schema_version(self):
+    return 1
+
   @property
   def content_type(self):
     return DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE

image/docker/schema2/list.py

@@ -0,0 +1,194 @@
+import json
+
+from cachetools import lru_cache
+from jsonschema import validate as validate_schema, ValidationError
+
+from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
+from image.docker.schema1 import DockerSchema1Manifest
+from image.docker.schema2 import (DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
+                                  DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
+from image.docker.schema2.manifest import DockerSchema2Manifest
+
+# Keys.
+DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY = 'schemaVersion'
+DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY = 'mediaType'
+DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY = 'size'
+DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY = 'digest'
+DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY = 'manifests'
+DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY = 'platform'
+DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY = 'architecture'
+DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY = 'os'
+DOCKER_SCHEMA2_MANIFESTLIST_OS_VERSION_KEY = 'os.version'
+DOCKER_SCHEMA2_MANIFESTLIST_OS_FEATURES_KEY = 'os.features'
+DOCKER_SCHEMA2_MANIFESTLIST_FEATURES_KEY = 'features'
+DOCKER_SCHEMA2_MANIFESTLIST_VARIANT_KEY = 'variant'
+
+
+class MalformedSchema2ManifestList(Exception):
+  """
+  Raised when a manifest list fails an assertion that should be true according to the
+  Docker Manifest v2.2 Specification.
+  """
+  pass
+
+
+class LazyManifestLoader(object):
+  def __init__(self, manifest_data, lookup_manifest_fn):
+    self._manifest_data = manifest_data
+    self._lookup_manifest_fn = lookup_manifest_fn
+    self._loaded_manifest = None
+
+  @property
+  def manifest_obj(self):
+    if self._loaded_manifest is not None:
+      return self._loaded_manifest
+
+    self._loaded_manifest = self._load_manifest()
+    return self._loaded_manifest
+
+  def _load_manifest(self):
+    digest = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY]
+    size = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY]
+    manifest_bytes = self._lookup_manifest_fn(digest)
+    if len(manifest_bytes) != size:
+      raise MalformedSchema2ManifestList('Size of manifest does not match that retrieved: %s vs %s',
+                                         len(manifest_bytes), size)
+
+    content_type = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY]
+    if content_type == DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE:
+      return DockerSchema2Manifest(manifest_bytes)
+
+    if content_type == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE:
+      return DockerSchema1Manifest(manifest_bytes, validate=False)
+
+    raise MalformedSchema2ManifestList('Unknown manifest content type')
+
+
+class DockerSchema2ManifestList(object):
+  METASCHEMA = {
+    'type': 'object',
+    'properties': {
+      DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY: {
+        'type': 'number',
+        'description': 'The version of the manifest list. Must always be `2`.',
+        'minimum': 2,
+        'maximum': 2,
+      },
+      DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: {
+        'type': 'string',
+        'description': 'The media type of the manifest list.',
+        'enum': [DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE],
+      },
+      DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY: {
+        'type': 'array',
+        'description': 'The manifests field contains a list of manifests for specific platforms',
+        'items': {
+          'type': 'object',
+          'properties': {
+            DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: {
+              'type': 'string',
+              'description': 'The MIME type of the referenced object. This will generally be ' +
+                             'application/vnd.docker.distribution.manifest.v2+json, but it ' +
+                             'could also be application/vnd.docker.distribution.manifest.v1+json ' +
+                             'if the manifest list references a legacy schema-1 manifest.',
+              'enum': [DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE],
+            },
+            DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY: {
+              'type': 'number',
+              'description': 'The size in bytes of the object. This field exists so that a ' +
+                             'client will have an expected size for the content before ' +
+                             'validating. If the length of the retrieved content does not ' +
+                             'match the specified length, the content should not be trusted.',
+            },
+            DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY: {
+              'type': 'string',
+              'description': 'The content addressable digest of the manifest in the blob store',
+            },
+            DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY: {
+              'type': 'object',
+              'description': 'The platform object describes the platform which the image in ' +
+                             'the manifest runs on',
+              'properties': {
+                DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY: {
+                  'type': 'string',
+                  'description': 'Specifies the CPU architecture, for example amd64 or ppc64le.',
+                },
+                DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY: {
+                  'type': 'string',
+                  'description': 'Specifies the operating system, for example linux or windows',
+                },
+                DOCKER_SCHEMA2_MANIFESTLIST_OS_VERSION_KEY: {
+                  'type': 'string',
+                  'description': 'Specifies the operating system version, for example 10.0.10586',
+                },
+                DOCKER_SCHEMA2_MANIFESTLIST_OS_FEATURES_KEY: {
+                  'type': 'array',
+                  'description': 'specifies an array of strings, each listing a required OS ' +
+                                 'feature (for example on Windows win32k)',
+                  'items': {
+                    'type': 'string',
+                  },
+                },
+                DOCKER_SCHEMA2_MANIFESTLIST_VARIANT_KEY: {
+                  'type': 'string',
+                  'description': 'Specifies a variant of the CPU, for example armv6l to specify ' +
+                                 'a particular CPU variant of the ARM CPU',
+                },
+                DOCKER_SCHEMA2_MANIFESTLIST_FEATURES_KEY: {
+                  'type': 'array',
+                  'description': 'specifies an array of strings, each listing a required CPU ' +
+                                 'feature (for example sse4 or aes).',
+                  'items': {
+                    'type': 'string',
+                  },
+                },
+              },
+              'required': [DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY,
+                           DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY],
+            },
+          },
+          'required': [DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY,
+                       DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY,
+                       DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY,
+                       DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY],
+        },
+      },
+    },
+    'required': [DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY,
+                 DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY,
+                 DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY],
+  }
+
+  def __init__(self, manifest_bytes):
+    self._layers = None
+
+    try:
+      self._parsed = json.loads(manifest_bytes)
+    except ValueError as ve:
+      raise MalformedSchema2ManifestList('malformed manifest data: %s' % ve)
+
+    try:
+      validate_schema(self._parsed, DockerSchema2ManifestList.METASCHEMA)
+    except ValidationError as ve:
+      raise MalformedSchema2ManifestList('manifest data does not match schema: %s' % ve)
+
+  @lru_cache(maxsize=1)
+  def manifests(self, lookup_manifest_fn):
+    """ Returns the manifests in the list. The `lookup_manifest_fn` is a function
+        that returns the manifest bytes for the specified digest.
+    """
+    manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]
+    return [LazyManifestLoader(m, lookup_manifest_fn) for m in manifests]
+
+  def get_v1_compatible_manifest(self, lookup_manifest_fn):
+    """ Returns the manifest that is compatible with V1, by virtue of being `amd64` and `linux`.
+        If none, returns None.
+    """
+    for manifest in self.manifests(lookup_manifest_fn):
+      platform = manifest._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY]
+      architecture = platform[DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY]
+      os = platform[DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY]
+      if architecture == 'amd64' and os == 'linux':
+        return manifest
+
+    return None

image/docker/schema2/manifest.py

@@ -3,7 +3,6 @@ import logging
 import hashlib
 
 from collections import namedtuple
-
 from jsonschema import validate as validate_schema, ValidationError
 
 from digest import digest_tools
@@ -133,6 +132,10 @@ class DockerSchema2Manifest(object):
     except ValidationError as ve:
       raise MalformedSchema2Manifest('manifest data does not match schema: %s' % ve)
 
+  @property
+  def schema_version(self):
+    return 2
+
   @property
   def config(self):
     config = self._parsed[DOCKER_SCHEMA2_MANIFEST_CONFIG_KEY]
@@ -192,6 +195,10 @@ class DockerSchema2Manifest(object):
         digest SHA, returns the associated configuration JSON bytes for this schema.
     """
     config_bytes = lookup_config_fn(self.config.digest)
+    if len(config_bytes) != self.config.size:
+      raise MalformedSchema2Manifest('Size of config does not match that retrieved: %s vs %s',
+                                     len(config_bytes), self.config.size)
+
     schema2_config = DockerSchema2Config(config_bytes)
 
     # Build the V1 IDs for the layers.

image/docker/schema2/test/test_list.py

@@ -0,0 +1,70 @@
+import json
+import pytest
+
+from image.docker.schema1 import DockerSchema1Manifest
+from image.docker.schema2.manifest import DockerSchema2Manifest
+from image.docker.schema2.list import MalformedSchema2ManifestList, DockerSchema2ManifestList
+from image.docker.schema2.test.test_manifest import MANIFEST_BYTES as v22_bytes
+from image.docker.test.test_schema1 import MANIFEST_BYTES as v21_bytes
+
+@pytest.mark.parametrize('json_data', [
+  '',
+  '{}',
+  """
+  {
+    "unknown": "key"
+  }
+  """,
+])
+def test_malformed_manifest_lists(json_data):
+  with pytest.raises(MalformedSchema2ManifestList):
+    DockerSchema2ManifestList(json_data)
+
+
+MANIFESTLIST_BYTES = json.dumps({
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+  "manifests": [
+    {
+      "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+      "size": 983,
+      "digest": "sha256:e6",
+      "platform": {
+        "architecture": "ppc64le",
+        "os": "linux",
+      }
+    },
+    {
+      "mediaType": "application/vnd.docker.distribution.manifest.v1+json",
+      "size": 878,
+      "digest": "sha256:5b",
+      "platform": {
+        "architecture": "amd64",
+        "os": "linux",
+        "features": [
+          "sse4"
+        ]
+      }
+    }
+  ]
+})
+
+def test_valid_manifestlist():
+  def _get_manifest(digest):
+    if digest == 'sha256:e6':
+      return v22_bytes
+    else:
+      return v21_bytes
+
+  manifestlist = DockerSchema2ManifestList(MANIFESTLIST_BYTES)
+  assert len(manifestlist.manifests(_get_manifest)) == 2
+
+  for index, manifest in enumerate(manifestlist.manifests(_get_manifest)):
+    if index == 0:
+      assert isinstance(manifest.manifest_obj, DockerSchema2Manifest)
+      assert manifest.manifest_obj.schema_version == 2
+    else:
+      assert isinstance(manifest.manifest_obj, DockerSchema1Manifest)
+      assert manifest.manifest_obj.schema_version == 1
+
+  assert manifestlist.get_v1_compatible_manifest(_get_manifest).manifest_obj.schema_version == 1

image/docker/schema2/test/test_manifest.py

@@ -25,7 +25,7 @@ MANIFEST_BYTES = json.dumps({
   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
   "config": {
     "mediaType": "application/vnd.docker.container.image.v1+json",
-    "size": 7023,
+    "size": 1885,
     "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
   },
   "layers": [
@@ -55,7 +55,7 @@ MANIFEST_BYTES = json.dumps({
 
 def test_valid_manifest():
   manifest = DockerSchema2Manifest(MANIFEST_BYTES)
-  assert manifest.config.size == 7023
+  assert manifest.config.size == 1885
   assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
   
   assert len(manifest.layers) == 4

image/docker/test/test_schema1.py

@@ -17,53 +17,50 @@ def test_malformed_manifests(json_data):
     DockerSchema1Manifest(json_data)
 
 
-@pytest.mark.parametrize('namespace', [
+MANIFEST_BYTES = json.dumps({
-  '',
+  "name": 'hello-world',
-  'somenamespace',
+  "tag": "latest",
-])
+  "architecture": "amd64",
-def test_valid_manifest(namespace):
+  "fsLayers": [
-  manifest_bytes = json.dumps({
+    {
-    "name": namespace + "/hello-world" if namespace else 'hello-world',
+      "blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"
-    "tag": "latest",
+    },
-    "architecture": "amd64",
+    {
-    "fsLayers": [
+      "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
-      {
+    }
-        "blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"
+  ],
-      },
+  "history": [
-      {
+    {
-        "blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+      "v1Compatibility": "{\"id\":\"someid\", \"parent\": \"anotherid\"}"
-      }
+    },
-    ],
+    {
-    "history": [
+      "v1Compatibility": "{\"id\":\"anotherid\"}"
-      {
+    },
-        "v1Compatibility": "{\"id\":\"someid\", \"parent\": \"anotherid\"}"
+  ],
-      },
+  "schemaVersion": 1,
-      {
+  "signatures": [
-        "v1Compatibility": "{\"id\":\"anotherid\"}"
+    {
-      },
+      "header": {
-    ],
+        "jwk": {
-    "schemaVersion": 1,
+          "crv": "P-256",
-    "signatures": [
+          "kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4",
-      {
+          "kty": "EC",
-        "header": {
+          "x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A",
-          "jwk": {
+          "y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010"
-            "crv": "P-256",
-            "kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4",
-            "kty": "EC",
-            "x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A",
-            "y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010"
-          },
-          "alg": "ES256"
         },
-        "signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg",
+        "alg": "ES256"
-        "protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ"
+      },
-      }
+      "signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg",
-    ]
+      "protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ"
-  })
+    }
+  ]
+})
 
-  manifest = DockerSchema1Manifest(manifest_bytes, validate=False)
+
+def test_valid_manifest():
+  manifest = DockerSchema1Manifest(MANIFEST_BYTES, validate=False)
   assert len(manifest.signatures) == 1
-  assert manifest.namespace == namespace
+  assert manifest.namespace == ''
   assert manifest.repo_name == 'hello-world'
   assert manifest.tag == 'latest'
   assert manifest.image_ids == {'someid', 'anotherid'}