Add support for creating schema 2 manifests and manifest lists via the OCI model
This commit is contained in:
Joseph Schorr
parent
e344d4a5cf
commit
30f072aeff
16 changed files with 398 additions and 110 deletions
|
|
@ -1,15 +1,28 @@
|
|||
import logging
|
||||
|
||||
from collections import namedtuple
|
||||
|
||||
from peewee import IntegrityError
|
||||
|
||||
from data.database import Tag, Manifest, ManifestBlob, ManifestLegacyImage, db_transaction
|
||||
from data.database import (Tag, Manifest, ManifestBlob, ManifestLegacyImage, ManifestChild,
|
||||
db_transaction)
|
||||
from data.model import BlobDoesNotExist
|
||||
from data.model.oci.tag import filter_to_alive_tags
|
||||
from data.model.storage import lookup_repo_storages_by_content_checksum
|
||||
from data.model.oci.label import create_manifest_label
|
||||
from data.model.storage import (lookup_repo_storages_by_content_checksum, get_storage_locations,
|
||||
get_layer_path)
|
||||
from data.model.blob import get_repository_blob_by_digest
|
||||
from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
|
||||
from image.docker.schema1 import DockerSchema1Manifest, ManifestException
|
||||
from image.docker.schema1 import ManifestException
|
||||
from image.docker.schema2.list import MalformedSchema2ManifestList
|
||||
from util.validation import is_json
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CreatedManifest = namedtuple('CreatedManifest', ['manifest', 'newly_created', 'labels_to_apply'])
|
||||
|
||||
|
||||
def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
|
||||
""" Returns the manifest with the specified digest under the specified repository
|
||||
or None if none. If allow_dead is True, then manifests referenced by only
|
||||
|
|
@ -29,43 +42,97 @@ def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
|
|||
return None
|
||||
|
||||
|
||||
def get_or_create_manifest(repository_id, manifest_interface_instance):
|
||||
""" Returns a tuple of the manifest in the specified repository with the matching digest
|
||||
(if it already exists) or, if not yet created, creates and returns the manifest, as well as
|
||||
if the manifest was created. Returns (None, None) if there was an error creating the manifest.
|
||||
def get_or_create_manifest(repository_id, manifest_interface_instance, storage):
|
||||
""" Returns a CreatedManifest for the manifest in the specified repository with the matching
|
||||
digest (if it already exists) or, if not yet created, creates and returns the manifest.
|
||||
Returns None if there was an error creating the manifest.
|
||||
Note that *all* blobs referenced by the manifest must exist already in the repository or this
|
||||
method will fail with a (None, None).
|
||||
method will fail with a None.
|
||||
"""
|
||||
existing = lookup_manifest(repository_id, manifest_interface_instance.digest, allow_dead=True)
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
return CreatedManifest(manifest=existing, newly_created=False, labels_to_apply=None)
|
||||
|
||||
assert len(list(manifest_interface_instance.layers)) > 0
|
||||
return _create_manifest(repository_id, manifest_interface_instance, storage)
|
||||
|
||||
# TODO(jschorr): Switch this to supporting schema2 once we're ready.
|
||||
assert isinstance(manifest_interface_instance, DockerSchema1Manifest)
|
||||
|
||||
def _create_manifest(repository_id, manifest_interface_instance, storage):
|
||||
digests = set(manifest_interface_instance.blob_digests)
|
||||
|
||||
def _lookup_digest(digest):
|
||||
return _retrieve_bytes_in_storage(repository_id, digest, storage)
|
||||
|
||||
# Retrieve the child manifests, if any. If we do retrieve a child manifest, we also remove its
|
||||
# blob from the list of blobs for this manifest, as the blob isn't really a "blob".
|
||||
child_manifest_refs = manifest_interface_instance.child_manifests(_lookup_digest)
|
||||
child_manifest_rows = []
|
||||
child_manifest_label_dicts = []
|
||||
|
||||
if child_manifest_refs is not None:
|
||||
for child_manifest_ref in child_manifest_refs:
|
||||
# Load and parse the child manifest.
|
||||
try:
|
||||
child_manifest = child_manifest_ref.manifest_obj
|
||||
except ManifestException:
|
||||
logger.exception('Could not load manifest list for manifest `%s`',
|
||||
manifest_interface_instance.digest)
|
||||
return None
|
||||
except MalformedSchema2ManifestList:
|
||||
logger.exception('Could not load manifest list for manifest `%s`',
|
||||
manifest_interface_instance.digest)
|
||||
return None
|
||||
except BlobDoesNotExist:
|
||||
logger.exception('Could not load manifest list for manifest `%s`',
|
||||
manifest_interface_instance.digest)
|
||||
return None
|
||||
except IOError:
|
||||
logger.exception('Could not load manifest list for manifest `%s`',
|
||||
manifest_interface_instance.digest)
|
||||
return None
|
||||
|
||||
# Retrieve its labels.
|
||||
labels = child_manifest.get_manifest_labels(_lookup_digest)
|
||||
if labels is None:
|
||||
logger.exception('Could not load manifest labels for child manifest')
|
||||
return None
|
||||
|
||||
# Get/create the child manifest in the database.
|
||||
assert list(child_manifest.layers)
|
||||
child_manifest_info = get_or_create_manifest(repository_id, child_manifest, storage)
|
||||
if child_manifest_info is None:
|
||||
logger.error('Could not get/create child manifest')
|
||||
return None
|
||||
|
||||
child_manifest_rows.append(child_manifest_info.manifest)
|
||||
child_manifest_label_dicts.append(labels)
|
||||
digests.remove(child_manifest.digest)
|
||||
|
||||
# Ensure all the blobs in the manifest exist.
|
||||
digests = manifest_interface_instance.checksums
|
||||
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
|
||||
blob_map = {s.content_checksum: s for s in query}
|
||||
for digest_str in manifest_interface_instance.blob_digests:
|
||||
if digest_str not in blob_map:
|
||||
logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
|
||||
manifest_interface_instance.digest, repository_id)
|
||||
return None, None
|
||||
blob_map = {}
|
||||
if digests:
|
||||
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
|
||||
blob_map = {s.content_checksum: s for s in query}
|
||||
for digest_str in digests:
|
||||
if digest_str not in blob_map:
|
||||
logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
|
||||
manifest_interface_instance.digest, repository_id)
|
||||
return None
|
||||
|
||||
# Determine and populate the legacy image if necessary.
|
||||
legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map)
|
||||
if legacy_image_id is None:
|
||||
return None, None
|
||||
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
|
||||
# image.
|
||||
legacy_image = None
|
||||
if manifest_interface_instance.leaf_layer_v1_image_id is not None:
|
||||
legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map,
|
||||
storage)
|
||||
if legacy_image_id is None:
|
||||
return None
|
||||
|
||||
legacy_image = get_image(repository_id, legacy_image_id)
|
||||
if legacy_image is None:
|
||||
return None, None
|
||||
legacy_image = get_image(repository_id, legacy_image_id)
|
||||
if legacy_image is None:
|
||||
return None
|
||||
|
||||
# Create the manifest and its blobs.
|
||||
media_type = Manifest.media_type.get_id(manifest_interface_instance.content_type)
|
||||
media_type = Manifest.media_type.get_id(manifest_interface_instance.media_type)
|
||||
storage_ids = {storage.id for storage in blob_map.values()}
|
||||
|
||||
with db_transaction():
|
||||
|
|
@ -77,7 +144,7 @@ def get_or_create_manifest(repository_id, manifest_interface_instance):
|
|||
manifest_bytes=manifest_interface_instance.bytes)
|
||||
except IntegrityError:
|
||||
manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest)
|
||||
return manifest, False
|
||||
return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None)
|
||||
|
||||
# Insert the blobs.
|
||||
blobs_to_insert = [dict(manifest=manifest, repository=repository_id,
|
||||
|
|
@ -86,12 +153,42 @@ def get_or_create_manifest(repository_id, manifest_interface_instance):
|
|||
ManifestBlob.insert_many(blobs_to_insert).execute()
|
||||
|
||||
# Set the legacy image (if applicable).
|
||||
ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)
|
||||
if legacy_image is not None:
|
||||
ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)
|
||||
|
||||
return manifest, True
|
||||
# Insert the manifest child rows (if applicable).
|
||||
if child_manifest_rows:
|
||||
children_to_insert = [dict(manifest=manifest, child_manifest=child_manifest,
|
||||
repository=repository_id)
|
||||
for child_manifest in child_manifest_rows]
|
||||
ManifestChild.insert_many(children_to_insert).execute()
|
||||
|
||||
# Define the labels for the manifest (if any).
|
||||
labels = manifest_interface_instance.get_manifest_labels(_lookup_digest)
|
||||
if labels:
|
||||
for key, value in labels.iteritems():
|
||||
media_type = 'application/json' if is_json(value) else 'text/plain'
|
||||
create_manifest_label(manifest, key, value, 'manifest', media_type)
|
||||
|
||||
# Return the dictionary of labels to apply. We only return those labels either defined on
|
||||
# the manifest or shared amongst all the child manifest.
|
||||
labels_to_apply = labels or {}
|
||||
if child_manifest_label_dicts:
|
||||
labels_to_apply = child_manifest_label_dicts[0].viewitems()
|
||||
for child_manifest_label_dict in child_manifest_label_dicts[1:]:
|
||||
# Intersect the key+values of the labels to ensure we get the exact same result
|
||||
# for all the child manifests.
|
||||
labels_to_apply = labels_to_apply & child_manifest_label_dict.viewitems()
|
||||
|
||||
labels_to_apply = dict(labels_to_apply)
|
||||
|
||||
return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply)
|
||||
|
||||
|
||||
def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map):
|
||||
def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map, storage):
|
||||
def _lookup_digest(digest):
|
||||
return _retrieve_bytes_in_storage(repository_id, digest, storage)
|
||||
|
||||
# Lookup all the images and their parent images (if any) inside the manifest.
|
||||
# This will let us know which v1 images we need to synthesize and which ones are invalid.
|
||||
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
|
||||
|
|
@ -100,7 +197,8 @@ def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map)
|
|||
|
||||
# Rewrite any v1 image IDs that do not match the checksum in the database.
|
||||
try:
|
||||
rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(image_storage_map)
|
||||
rewritten_images = manifest_interface_instance.generate_legacy_layers(image_storage_map,
|
||||
_lookup_digest)
|
||||
rewritten_images = list(rewritten_images)
|
||||
parent_image_map = {}
|
||||
|
||||
|
|
@ -132,3 +230,12 @@ def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map)
|
|||
return None
|
||||
|
||||
return rewritten_images[-1].image_id
|
||||
|
||||
|
||||
def _retrieve_bytes_in_storage(repository_id, digest, storage):
|
||||
blob = get_repository_blob_by_digest(repository_id, digest)
|
||||
if blob is None:
|
||||
return None
|
||||
|
||||
placements = list(get_storage_locations(blob.uuid))
|
||||
return storage.get_content(placements, get_layer_path(blob))
|
||||
|
|
|
|||
Reference in a new issue