vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add schema2 list support

Browse source
This commit is contained in:
Joseph Schorr 2018-04-16 17:22:25 +03:00
parent 52b12131f7
commit a73cd9170a
7 changed files with 320 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
image/docker/schema1.py
View file

@ -195,6 +195,10 @@ class DockerSchema1Manifest(object):
if not verified:
raise InvalidSchema1Signature()
@property
def schema_version(self):
return 1
@property
def content_type(self):
return DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE

194
image/docker/schema2/list.py Normal file
View file

@ -0,0 +1,194 @@
import json
from cachetools import lru_cache
from jsonschema import validate as validate_schema, ValidationError
from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
from image.docker.schema1 import DockerSchema1Manifest
from image.docker.schema2 import (DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
from image.docker.schema2.manifest import DockerSchema2Manifest
# Keys.
DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY = 'schemaVersion'
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY = 'mediaType'
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY = 'size'
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY = 'digest'
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY = 'manifests'
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY = 'platform'
DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY = 'architecture'
DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY = 'os'
DOCKER_SCHEMA2_MANIFESTLIST_OS_VERSION_KEY = 'os.version'
DOCKER_SCHEMA2_MANIFESTLIST_OS_FEATURES_KEY = 'os.features'
DOCKER_SCHEMA2_MANIFESTLIST_FEATURES_KEY = 'features'
DOCKER_SCHEMA2_MANIFESTLIST_VARIANT_KEY = 'variant'
class MalformedSchema2ManifestList(Exception):
"""
Raised when a manifest list fails an assertion that should be true according to the
Docker Manifest v2.2 Specification.
"""
pass
class LazyManifestLoader(object):
def __init__(self, manifest_data, lookup_manifest_fn):
self._manifest_data = manifest_data
self._lookup_manifest_fn = lookup_manifest_fn
self._loaded_manifest = None
@property
def manifest_obj(self):
if self._loaded_manifest is not None:
return self._loaded_manifest
self._loaded_manifest = self._load_manifest()
return self._loaded_manifest
def _load_manifest(self):
digest = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY]
size = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY]
manifest_bytes = self._lookup_manifest_fn(digest)
if len(manifest_bytes) != size:
raise MalformedSchema2ManifestList('Size of manifest does not match that retrieved: %s vs %s',
len(manifest_bytes), size)
content_type = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY]
if content_type == DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE:
return DockerSchema2Manifest(manifest_bytes)
if content_type == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE:
return DockerSchema1Manifest(manifest_bytes, validate=False)
raise MalformedSchema2ManifestList('Unknown manifest content type')
class DockerSchema2ManifestList(object):
METASCHEMA = {
'type': 'object',
'properties': {
DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY: {
'type': 'number',
'description': 'The version of the manifest list. Must always be `2`.',
'minimum': 2,
'maximum': 2,
},
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: {
'type': 'string',
'description': 'The media type of the manifest list.',
'enum': [DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE],
},
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY: {
'type': 'array',
'description': 'The manifests field contains a list of manifests for specific platforms',
'items': {
'type': 'object',
'properties': {
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: {
'type': 'string',
'description': 'The MIME type of the referenced object. This will generally be ' +
'application/vnd.docker.distribution.manifest.v2+json, but it ' +
'could also be application/vnd.docker.distribution.manifest.v1+json ' +
'if the manifest list references a legacy schema-1 manifest.',
'enum': [DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE],
},
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY: {
'type': 'number',
'description': 'The size in bytes of the object. This field exists so that a ' +
'client will have an expected size for the content before ' +
'validating. If the length of the retrieved content does not ' +
'match the specified length, the content should not be trusted.',
},
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY: {
'type': 'string',
'description': 'The content addressable digest of the manifest in the blob store',
},
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY: {
'type': 'object',
'description': 'The platform object describes the platform which the image in ' +
'the manifest runs on',
'properties': {
DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY: {
'type': 'string',
'description': 'Specifies the CPU architecture, for example amd64 or ppc64le.',
},
DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY: {
'type': 'string',
'description': 'Specifies the operating system, for example linux or windows',
},
DOCKER_SCHEMA2_MANIFESTLIST_OS_VERSION_KEY: {
'type': 'string',
'description': 'Specifies the operating system version, for example 10.0.10586',
},
DOCKER_SCHEMA2_MANIFESTLIST_OS_FEATURES_KEY: {
'type': 'array',
'description': 'specifies an array of strings, each listing a required OS ' +
'feature (for example on Windows win32k)',
'items': {
'type': 'string',
},
},
DOCKER_SCHEMA2_MANIFESTLIST_VARIANT_KEY: {
'type': 'string',
'description': 'Specifies a variant of the CPU, for example armv6l to specify ' +
'a particular CPU variant of the ARM CPU',
},
DOCKER_SCHEMA2_MANIFESTLIST_FEATURES_KEY: {
'type': 'array',
'description': 'specifies an array of strings, each listing a required CPU ' +
'feature (for example sse4 or aes).',
'items': {
'type': 'string',
},
},
},
'required': [DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY,
DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY],
},
},
'required': [DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY,
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY,
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY,
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY],
},
},
},
'required': [DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY,
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY,
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY],
}
def __init__(self, manifest_bytes):
self._layers = None
try:
self._parsed = json.loads(manifest_bytes)
except ValueError as ve:
raise MalformedSchema2ManifestList('malformed manifest data: %s' % ve)
try:
validate_schema(self._parsed, DockerSchema2ManifestList.METASCHEMA)
except ValidationError as ve:
raise MalformedSchema2ManifestList('manifest data does not match schema: %s' % ve)
@lru_cache(maxsize=1)
def manifests(self, lookup_manifest_fn):
""" Returns the manifests in the list. The `lookup_manifest_fn` is a function
that returns the manifest bytes for the specified digest.
"""
manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]
return [LazyManifestLoader(m, lookup_manifest_fn) for m in manifests]
def get_v1_compatible_manifest(self, lookup_manifest_fn):
""" Returns the manifest that is compatible with V1, by virtue of being `amd64` and `linux`.
If none, returns None.
"""
for manifest in self.manifests(lookup_manifest_fn):
platform = manifest._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY]
architecture = platform[DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY]
os = platform[DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY]
if architecture == 'amd64' and os == 'linux':
return manifest
return None

11
image/docker/schema2/manifest.py
View file

@ -3,7 +3,6 @@ import logging
import hashlib
from collections import namedtuple
from jsonschema import validate as validate_schema, ValidationError
from digest import digest_tools
@ -133,6 +132,10 @@ class DockerSchema2Manifest(object):
except ValidationError as ve:
raise MalformedSchema2Manifest('manifest data does not match schema: %s' % ve)
@property
def schema_version(self):
return 2
@property
def config(self):
config = self._parsed[DOCKER_SCHEMA2_MANIFEST_CONFIG_KEY]
@ -160,7 +163,7 @@ class DockerSchema2Manifest(object):
except digest_tools.InvalidDigestException:
raise MalformedSchema2Manifest('could not parse manifest digest: %s' %
layer[DOCKER_SCHEMA2_MANIFEST_DIGEST_KEY])
yield DockerV2ManifestLayer(index=index,
size=layer[DOCKER_SCHEMA2_MANIFEST_SIZE_KEY],
digest=digest,
@ -192,6 +195,10 @@ class DockerSchema2Manifest(object):
digest SHA, returns the associated configuration JSON bytes for this schema.
"""
config_bytes = lookup_config_fn(self.config.digest)
if len(config_bytes) != self.config.size:
raise MalformedSchema2Manifest('Size of config does not match that retrieved: %s vs %s',
len(config_bytes), self.config.size)
schema2_config = DockerSchema2Config(config_bytes)
# Build the V1 IDs for the layers.

70
image/docker/schema2/test/test_list.py Normal file
View file

@ -0,0 +1,70 @@
import json
import pytest
from image.docker.schema1 import DockerSchema1Manifest
from image.docker.schema2.manifest import DockerSchema2Manifest
from image.docker.schema2.list import MalformedSchema2ManifestList, DockerSchema2ManifestList
from image.docker.schema2.test.test_manifest import MANIFEST_BYTES as v22_bytes
from image.docker.test.test_schema1 import MANIFEST_BYTES as v21_bytes
@pytest.mark.parametrize('json_data', [
'',
'{}',
"""
{
"unknown": "key"
}
""",
])
def test_malformed_manifest_lists(json_data):
with pytest.raises(MalformedSchema2ManifestList):
DockerSchema2ManifestList(json_data)
MANIFESTLIST_BYTES = json.dumps({
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
"manifests": [
{
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"size": 983,
"digest": "sha256:e6",
"platform": {
"architecture": "ppc64le",
"os": "linux",
}
},
{
"mediaType": "application/vnd.docker.distribution.manifest.v1+json",
"size": 878,
"digest": "sha256:5b",
"platform": {
"architecture": "amd64",
"os": "linux",
"features": [
"sse4"
]
}
}
]
})
def test_valid_manifestlist():
def _get_manifest(digest):
if digest == 'sha256:e6':
return v22_bytes
else:
return v21_bytes
manifestlist = DockerSchema2ManifestList(MANIFESTLIST_BYTES)
assert len(manifestlist.manifests(_get_manifest)) == 2
for index, manifest in enumerate(manifestlist.manifests(_get_manifest)):
if index == 0:
assert isinstance(manifest.manifest_obj, DockerSchema2Manifest)
assert manifest.manifest_obj.schema_version == 2
else:
assert isinstance(manifest.manifest_obj, DockerSchema1Manifest)
assert manifest.manifest_obj.schema_version == 1
assert manifestlist.get_v1_compatible_manifest(_get_manifest).manifest_obj.schema_version == 1

4
image/docker/schema2/test/test_manifest.py
View file

@ -25,7 +25,7 @@ MANIFEST_BYTES = json.dumps({
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": 7023,
"size": 1885,
"digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
},
"layers": [
@ -55,7 +55,7 @@ MANIFEST_BYTES = json.dumps({
def test_valid_manifest():
manifest = DockerSchema2Manifest(MANIFEST_BYTES)
assert manifest.config.size == 7023
assert manifest.config.size == 1885
assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
assert len(manifest.layers) == 4

0
image/docker/test/__init__.py Normal file
View file

85
image/docker/test/test_schema1.py
View file

@ -17,53 +17,50 @@ def test_malformed_manifests(json_data):
DockerSchema1Manifest(json_data)
@pytest.mark.parametrize('namespace', [
'',
'somenamespace',
])
def test_valid_manifest(namespace):
manifest_bytes = json.dumps({
"name": namespace + "/hello-world" if namespace else 'hello-world',
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"
},
{
"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"someid\", \"parent\": \"anotherid\"}"
},
{
"v1Compatibility": "{\"id\":\"anotherid\"}"
},
],
"schemaVersion": 1,
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4",
"kty": "EC",
"x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A",
"y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010"
},
"alg": "ES256"
MANIFEST_BYTES = json.dumps({
"name": 'hello-world',
"tag": "latest",
"architecture": "amd64",
"fsLayers": [
{
"blobSum": "sha256:cc8567d70002e957612902a8e985ea129d831ebe04057d88fb644857caa45d11"
},
{
"blobSum": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
}
],
"history": [
{
"v1Compatibility": "{\"id\":\"someid\", \"parent\": \"anotherid\"}"
},
{
"v1Compatibility": "{\"id\":\"anotherid\"}"
},
],
"schemaVersion": 1,
"signatures": [
{
"header": {
"jwk": {
"crv": "P-256",
"kid": "OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4",
"kty": "EC",
"x": "3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A",
"y": "t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010"
},
"signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg",
"protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ"
}
]
})
"alg": "ES256"
},
"signature": "XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg",
"protected": "eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ"
}
]
})
manifest = DockerSchema1Manifest(manifest_bytes, validate=False)
def test_valid_manifest():
manifest = DockerSchema1Manifest(MANIFEST_BYTES, validate=False)
assert len(manifest.signatures) == 1
assert manifest.namespace == namespace
assert manifest.namespace == ''
assert manifest.repo_name == 'hello-world'
assert manifest.tag == 'latest'
assert manifest.image_ids == {'someid', 'anotherid'}