initial import for Open Source 🎉
This commit is contained in:
Jimmy Zelinskie
parent
1898c361f3
commit
9c0dd3b722
2048 changed files with 218743 additions and 0 deletions
379
image/docker/schema2/list.py
Normal file
379
image/docker/schema2/list.py
Normal file
|
|
@ -0,0 +1,379 @@
|
|||
import logging
|
||||
import json
|
||||
|
||||
from cachetools.func import lru_cache
|
||||
from jsonschema import validate as validate_schema, ValidationError
|
||||
|
||||
from digest import digest_tools
|
||||
from image.docker import ManifestException
|
||||
from image.docker.interfaces import ManifestInterface
|
||||
from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
|
||||
from image.docker.schema1 import DockerSchema1Manifest
|
||||
from image.docker.schema2 import (DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
|
||||
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
|
||||
from image.docker.schema2.manifest import DockerSchema2Manifest
|
||||
from util.bytes import Bytes
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keys.
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY = 'schemaVersion'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY = 'mediaType'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY = 'size'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY = 'digest'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY = 'manifests'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY = 'platform'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY = 'architecture'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY = 'os'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_VERSION_KEY = 'os.version'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_FEATURES_KEY = 'os.features'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_FEATURES_KEY = 'features'
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_VARIANT_KEY = 'variant'
|
||||
|
||||
|
||||
class MalformedSchema2ManifestList(ManifestException):
|
||||
"""
|
||||
Raised when a manifest list fails an assertion that should be true according to the
|
||||
Docker Manifest v2.2 Specification.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class MismatchManifestException(MalformedSchema2ManifestList):
|
||||
""" Raised when a manifest list contains a schema 1 manifest with a differing architecture
|
||||
from that specified in the manifest list for the manifest.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class LazyManifestLoader(object):
|
||||
def __init__(self, manifest_data, content_retriever):
|
||||
self._manifest_data = manifest_data
|
||||
self._content_retriever = content_retriever
|
||||
self._loaded_manifest = None
|
||||
|
||||
@property
|
||||
def manifest_obj(self):
|
||||
if self._loaded_manifest is not None:
|
||||
return self._loaded_manifest
|
||||
|
||||
self._loaded_manifest = self._load_manifest()
|
||||
return self._loaded_manifest
|
||||
|
||||
def _load_manifest(self):
|
||||
digest = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY]
|
||||
size = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY]
|
||||
manifest_bytes = self._content_retriever.get_manifest_bytes_with_digest(digest)
|
||||
if manifest_bytes is None:
|
||||
raise MalformedSchema2ManifestList('Could not find child manifest with digest `%s`' % digest)
|
||||
|
||||
if len(manifest_bytes) != size:
|
||||
raise MalformedSchema2ManifestList('Size of manifest does not match that retrieved: %s vs %s',
|
||||
len(manifest_bytes), size)
|
||||
|
||||
content_type = self._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY]
|
||||
if content_type == DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE:
|
||||
return DockerSchema2Manifest(Bytes.for_string_or_unicode(manifest_bytes))
|
||||
|
||||
if content_type == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE:
|
||||
return DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes), validate=False)
|
||||
|
||||
raise MalformedSchema2ManifestList('Unknown manifest content type')
|
||||
|
||||
|
||||
class DockerSchema2ManifestList(ManifestInterface):
|
||||
METASCHEMA = {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY: {
|
||||
'type': 'number',
|
||||
'description': 'The version of the manifest list. Must always be `2`.',
|
||||
'minimum': 2,
|
||||
'maximum': 2,
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: {
|
||||
'type': 'string',
|
||||
'description': 'The media type of the manifest list.',
|
||||
'enum': [DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE],
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY: {
|
||||
'type': 'array',
|
||||
'description': 'The manifests field contains a list of manifests for specific platforms',
|
||||
'items': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: {
|
||||
'type': 'string',
|
||||
'description': 'The MIME type of the referenced object. This will generally be ' +
|
||||
'application/vnd.docker.distribution.manifest.v2+json, but it ' +
|
||||
'could also be application/vnd.docker.distribution.manifest.v1+json ' +
|
||||
'if the manifest list references a legacy schema-1 manifest.',
|
||||
'enum': [DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE, DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE],
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY: {
|
||||
'type': 'number',
|
||||
'description': 'The size in bytes of the object. This field exists so that a ' +
|
||||
'client will have an expected size for the content before ' +
|
||||
'validating. If the length of the retrieved content does not ' +
|
||||
'match the specified length, the content should not be trusted.',
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY: {
|
||||
'type': 'string',
|
||||
'description': 'The content addressable digest of the manifest in the blob store',
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY: {
|
||||
'type': 'object',
|
||||
'description': 'The platform object describes the platform which the image in ' +
|
||||
'the manifest runs on',
|
||||
'properties': {
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY: {
|
||||
'type': 'string',
|
||||
'description': 'Specifies the CPU architecture, for example amd64 or ppc64le.',
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY: {
|
||||
'type': 'string',
|
||||
'description': 'Specifies the operating system, for example linux or windows',
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_VERSION_KEY: {
|
||||
'type': 'string',
|
||||
'description': 'Specifies the operating system version, for example 10.0.10586',
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_FEATURES_KEY: {
|
||||
'type': 'array',
|
||||
'description': 'specifies an array of strings, each listing a required OS ' +
|
||||
'feature (for example on Windows win32k)',
|
||||
'items': {
|
||||
'type': 'string',
|
||||
},
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_VARIANT_KEY: {
|
||||
'type': 'string',
|
||||
'description': 'Specifies a variant of the CPU, for example armv6l to specify ' +
|
||||
'a particular CPU variant of the ARM CPU',
|
||||
},
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_FEATURES_KEY: {
|
||||
'type': 'array',
|
||||
'description': 'specifies an array of strings, each listing a required CPU ' +
|
||||
'feature (for example sse4 or aes).',
|
||||
'items': {
|
||||
'type': 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
'required': [DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY],
|
||||
},
|
||||
},
|
||||
'required': [DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY],
|
||||
},
|
||||
},
|
||||
},
|
||||
'required': [DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY],
|
||||
}
|
||||
|
||||
def __init__(self, manifest_bytes):
|
||||
assert isinstance(manifest_bytes, Bytes)
|
||||
|
||||
self._layers = None
|
||||
self._manifest_bytes = manifest_bytes
|
||||
|
||||
try:
|
||||
self._parsed = json.loads(manifest_bytes.as_unicode())
|
||||
except ValueError as ve:
|
||||
raise MalformedSchema2ManifestList('malformed manifest data: %s' % ve)
|
||||
|
||||
try:
|
||||
validate_schema(self._parsed, DockerSchema2ManifestList.METASCHEMA)
|
||||
except ValidationError as ve:
|
||||
raise MalformedSchema2ManifestList('manifest data does not match schema: %s' % ve)
|
||||
|
||||
@property
|
||||
def is_manifest_list(self):
|
||||
""" Returns whether this manifest is a list. """
|
||||
return True
|
||||
|
||||
@property
|
||||
def schema_version(self):
|
||||
return 2
|
||||
|
||||
@property
|
||||
def digest(self):
|
||||
""" The digest of the manifest, including type prefix. """
|
||||
return digest_tools.sha256_digest(self._manifest_bytes.as_encoded_str())
|
||||
|
||||
@property
|
||||
def media_type(self):
|
||||
""" The media type of the schema. """
|
||||
return self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY]
|
||||
|
||||
@property
|
||||
def manifest_dict(self):
|
||||
""" Returns the manifest as a dictionary ready to be serialized to JSON. """
|
||||
return self._parsed
|
||||
|
||||
@property
|
||||
def bytes(self):
|
||||
return self._manifest_bytes
|
||||
|
||||
def get_layers(self, content_retriever):
|
||||
""" Returns the layers of this manifest, from base to leaf or None if this kind of manifest
|
||||
does not support layers. """
|
||||
return None
|
||||
|
||||
@property
|
||||
def blob_digests(self):
|
||||
# Manifest lists have no blob digests, since everything is stored as a manifest.
|
||||
return []
|
||||
|
||||
@property
|
||||
def local_blob_digests(self):
|
||||
return self.blob_digests
|
||||
|
||||
def get_blob_digests_for_translation(self):
|
||||
return self.blob_digests
|
||||
|
||||
@property
|
||||
def layers_compressed_size(self):
|
||||
return None
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def manifests(self, content_retriever):
|
||||
""" Returns the manifests in the list.
|
||||
"""
|
||||
manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]
|
||||
return [LazyManifestLoader(m, content_retriever) for m in manifests]
|
||||
|
||||
def validate(self, content_retriever):
|
||||
""" Performs validation of required assertions about the manifest. Raises a ManifestException
|
||||
on failure.
|
||||
"""
|
||||
for index, m in enumerate(self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]):
|
||||
if m[DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY] == DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE:
|
||||
platform = m[DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY]
|
||||
|
||||
# Validate the architecture against the schema 1 architecture defined.
|
||||
parsed = self.manifests(content_retriever)[index].manifest_obj
|
||||
assert isinstance(parsed, DockerSchema1Manifest)
|
||||
if (parsed.architecture and
|
||||
parsed.architecture != platform[DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY]):
|
||||
raise MismatchManifestException('Mismatch in arch for manifest `%s`' % parsed.digest)
|
||||
|
||||
def child_manifests(self, content_retriever):
|
||||
return self.manifests(content_retriever)
|
||||
|
||||
def child_manifest_digests(self):
|
||||
return [m[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY]
|
||||
for m in self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]]
|
||||
|
||||
def get_manifest_labels(self, content_retriever):
|
||||
return None
|
||||
|
||||
def get_leaf_layer_v1_image_id(self, content_retriever):
|
||||
return None
|
||||
|
||||
def get_legacy_image_ids(self, content_retriever):
|
||||
return None
|
||||
|
||||
@property
|
||||
def has_legacy_image(self):
|
||||
return False
|
||||
|
||||
def get_requires_empty_layer_blob(self, content_retriever):
|
||||
return False
|
||||
|
||||
def get_schema1_manifest(self, namespace_name, repo_name, tag_name, content_retriever):
|
||||
""" Returns the manifest that is compatible with V1, by virtue of being `amd64` and `linux`.
|
||||
If none, returns None.
|
||||
"""
|
||||
legacy_manifest = self._get_legacy_manifest(content_retriever)
|
||||
if legacy_manifest is None:
|
||||
return None
|
||||
|
||||
return legacy_manifest.get_schema1_manifest(namespace_name, repo_name, tag_name,
|
||||
content_retriever)
|
||||
|
||||
def convert_manifest(self, allowed_mediatypes, namespace_name, repo_name, tag_name,
|
||||
content_retriever):
|
||||
if self.media_type in allowed_mediatypes:
|
||||
return self
|
||||
|
||||
legacy_manifest = self._get_legacy_manifest(content_retriever)
|
||||
if legacy_manifest is None:
|
||||
return None
|
||||
|
||||
return legacy_manifest.convert_manifest(allowed_mediatypes, namespace_name, repo_name,
|
||||
tag_name, content_retriever)
|
||||
|
||||
def _get_legacy_manifest(self, content_retriever):
|
||||
""" Returns the manifest under this list with architecture amd64 and os linux, if any, or None
|
||||
if none or error.
|
||||
"""
|
||||
for manifest_ref in self.manifests(content_retriever):
|
||||
platform = manifest_ref._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY]
|
||||
architecture = platform[DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY]
|
||||
os = platform[DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY]
|
||||
if architecture != 'amd64' or os != 'linux':
|
||||
continue
|
||||
|
||||
try:
|
||||
return manifest_ref.manifest_obj
|
||||
except (ManifestException, IOError):
|
||||
logger.exception('Could not load child manifest')
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def unsigned(self):
|
||||
return self
|
||||
|
||||
def generate_legacy_layers(self, images_map, content_retriever):
|
||||
return None
|
||||
|
||||
|
||||
class DockerSchema2ManifestListBuilder(object):
|
||||
"""
|
||||
A convenient abstraction around creating new DockerSchema2ManifestList's.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.manifests = []
|
||||
|
||||
def add_manifest(self, manifest, architecture, os):
|
||||
""" Adds a manifest to the list. """
|
||||
manifest = manifest.unsigned() # Make sure we add the unsigned version to the list.
|
||||
self.add_manifest_digest(manifest.digest,
|
||||
len(manifest.bytes.as_encoded_str()),
|
||||
manifest.media_type,
|
||||
architecture, os)
|
||||
|
||||
def add_manifest_digest(self, manifest_digest, manifest_size, media_type, architecture, os):
|
||||
""" Adds a manifest to the list. """
|
||||
self.manifests.append((manifest_digest, manifest_size, media_type, {
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY: architecture,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY: os,
|
||||
}))
|
||||
|
||||
def build(self):
|
||||
""" Builds and returns the DockerSchema2ManifestList. """
|
||||
assert self.manifests
|
||||
|
||||
manifest_list_dict = {
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_VERSION_KEY: 2,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY: [
|
||||
{
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_MEDIATYPE_KEY: manifest[2],
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY: manifest[0],
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_SIZE_KEY: manifest[1],
|
||||
DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY: manifest[3],
|
||||
} for manifest in self.manifests
|
||||
],
|
||||
}
|
||||
|
||||
json_str = Bytes.for_string_or_unicode(json.dumps(manifest_list_dict, indent=3))
|
||||
return DockerSchema2ManifestList(json_str)
|
||||
Reference in a new issue