234 lines
10 KiB
Python
234 lines
10 KiB
Python
import logging
|
|
|
|
from collections import namedtuple
|
|
|
|
from peewee import IntegrityError, JOIN
|
|
|
|
from data.database import (Tag, Manifest, ManifestBlob, ManifestLegacyImage, ManifestChild,
|
|
db_transaction)
|
|
from data.model import BlobDoesNotExist
|
|
from data.model.oci.tag import filter_to_alive_tags
|
|
from data.model.oci.label import create_manifest_label
|
|
from data.model.oci.retriever import RepositoryContentRetriever
|
|
from data.model.storage import lookup_repo_storages_by_content_checksum
|
|
from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
|
|
from image.docker.schema1 import ManifestException
|
|
from image.docker.schema2.list import MalformedSchema2ManifestList
|
|
from util.validation import is_json
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
CreatedManifest = namedtuple('CreatedManifest', ['manifest', 'newly_created', 'labels_to_apply'])
|
|
|
|
|
|
def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
|
|
""" Returns the manifest with the specified digest under the specified repository
|
|
or None if none. If allow_dead is True, then manifests referenced by only
|
|
dead tags will also be returned.
|
|
"""
|
|
query = (Manifest
|
|
.select()
|
|
.where(Manifest.repository == repository_id)
|
|
.where(Manifest.digest == manifest_digest))
|
|
|
|
if allow_dead:
|
|
try:
|
|
return query.get()
|
|
except Manifest.DoesNotExist:
|
|
return None
|
|
|
|
# Try first to filter to those manifests referenced by an alive tag,
|
|
try:
|
|
return filter_to_alive_tags(query.join(Tag)).get()
|
|
except Manifest.DoesNotExist:
|
|
pass
|
|
|
|
# Try referenced as the child of a manifest that has an alive tag.
|
|
query = (query
|
|
.join(ManifestChild, on=(ManifestChild.child_manifest == Manifest.id))
|
|
.join(Tag, on=(Tag.manifest == ManifestChild.manifest)))
|
|
|
|
query = filter_to_alive_tags(query)
|
|
|
|
try:
|
|
return query.get()
|
|
except Manifest.DoesNotExist:
|
|
return None
|
|
|
|
|
|
def get_or_create_manifest(repository_id, manifest_interface_instance, storage):
|
|
""" Returns a CreatedManifest for the manifest in the specified repository with the matching
|
|
digest (if it already exists) or, if not yet created, creates and returns the manifest.
|
|
Returns None if there was an error creating the manifest.
|
|
Note that *all* blobs referenced by the manifest must exist already in the repository or this
|
|
method will fail with a None.
|
|
"""
|
|
existing = lookup_manifest(repository_id, manifest_interface_instance.digest, allow_dead=True)
|
|
if existing is not None:
|
|
return CreatedManifest(manifest=existing, newly_created=False, labels_to_apply=None)
|
|
|
|
return _create_manifest(repository_id, manifest_interface_instance, storage)
|
|
|
|
|
|
def _create_manifest(repository_id, manifest_interface_instance, storage):
|
|
# Load, parse and get/create the child manifests, if any.
|
|
retriever = RepositoryContentRetriever.for_repository(repository_id, storage)
|
|
child_manifest_refs = manifest_interface_instance.child_manifests(retriever)
|
|
child_manifest_rows = {}
|
|
child_manifest_label_dicts = []
|
|
|
|
if child_manifest_refs is not None:
|
|
for child_manifest_ref in child_manifest_refs:
|
|
# Load and parse the child manifest.
|
|
try:
|
|
child_manifest = child_manifest_ref.manifest_obj
|
|
except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError):
|
|
logger.exception('Could not load manifest list for manifest `%s`',
|
|
manifest_interface_instance.digest)
|
|
return None
|
|
|
|
# Retrieve its labels.
|
|
labels = child_manifest.get_manifest_labels(retriever)
|
|
if labels is None:
|
|
logger.exception('Could not load manifest labels for child manifest')
|
|
return None
|
|
|
|
# NOTE: Content type restrictions in manifest lists ensure that the child manifests
|
|
# must be image manifests, as opposed to lists themselves. We put this check here to
|
|
# be extra careful in ensuring we don't create empty manifests. If this reality changes,
|
|
# should remove this check.
|
|
assert list(child_manifest.layers)
|
|
|
|
# Get/create the child manifest in the database.
|
|
child_manifest_info = get_or_create_manifest(repository_id, child_manifest, storage)
|
|
if child_manifest_info is None:
|
|
logger.error('Could not get/create child manifest')
|
|
return None
|
|
|
|
child_manifest_rows[child_manifest_info.manifest.digest] = child_manifest_info.manifest
|
|
child_manifest_label_dicts.append(labels)
|
|
|
|
# Ensure all the blobs in the manifest exist.
|
|
digests = set(manifest_interface_instance.local_blob_digests)
|
|
blob_map = {}
|
|
if digests:
|
|
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
|
|
blob_map = {s.content_checksum: s for s in query}
|
|
for digest_str in digests:
|
|
if digest_str not in blob_map:
|
|
logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
|
|
manifest_interface_instance.digest, repository_id)
|
|
return None
|
|
|
|
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
|
|
# image.
|
|
legacy_image = None
|
|
if manifest_interface_instance.leaf_layer_v1_image_id is not None:
|
|
legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map,
|
|
retriever)
|
|
if legacy_image_id is None:
|
|
return None
|
|
|
|
legacy_image = get_image(repository_id, legacy_image_id)
|
|
if legacy_image is None:
|
|
return None
|
|
|
|
# Create the manifest and its blobs.
|
|
media_type = Manifest.media_type.get_id(manifest_interface_instance.media_type)
|
|
storage_ids = {storage.id for storage in blob_map.values()}
|
|
|
|
with db_transaction():
|
|
# Create the manifest.
|
|
try:
|
|
manifest = Manifest.create(repository=repository_id,
|
|
digest=manifest_interface_instance.digest,
|
|
media_type=media_type,
|
|
manifest_bytes=manifest_interface_instance.bytes)
|
|
except IntegrityError:
|
|
manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest)
|
|
return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None)
|
|
|
|
# Insert the blobs.
|
|
blobs_to_insert = [dict(manifest=manifest, repository=repository_id,
|
|
blob=storage_id) for storage_id in storage_ids]
|
|
if blobs_to_insert:
|
|
ManifestBlob.insert_many(blobs_to_insert).execute()
|
|
|
|
# Set the legacy image (if applicable).
|
|
if legacy_image is not None:
|
|
ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)
|
|
|
|
# Insert the manifest child rows (if applicable).
|
|
if child_manifest_rows:
|
|
children_to_insert = [dict(manifest=manifest, child_manifest=child_manifest,
|
|
repository=repository_id)
|
|
for child_manifest in child_manifest_rows.values()]
|
|
ManifestChild.insert_many(children_to_insert).execute()
|
|
|
|
# Define the labels for the manifest (if any).
|
|
labels = manifest_interface_instance.get_manifest_labels(retriever)
|
|
if labels:
|
|
for key, value in labels.iteritems():
|
|
media_type = 'application/json' if is_json(value) else 'text/plain'
|
|
create_manifest_label(manifest, key, value, 'manifest', media_type)
|
|
|
|
# Return the dictionary of labels to apply (i.e. those labels that cause an action to be taken
|
|
# on the manifest or its resulting tags). We only return those labels either defined on
|
|
# the manifest or shared amongst all the child manifests. We intersect amongst all child manifests
|
|
# to ensure that any action performed is defined in all manifests.
|
|
labels_to_apply = labels or {}
|
|
if child_manifest_label_dicts:
|
|
labels_to_apply = child_manifest_label_dicts[0].viewitems()
|
|
for child_manifest_label_dict in child_manifest_label_dicts[1:]:
|
|
# Intersect the key+values of the labels to ensure we get the exact same result
|
|
# for all the child manifests.
|
|
labels_to_apply = labels_to_apply & child_manifest_label_dict.viewitems()
|
|
|
|
labels_to_apply = dict(labels_to_apply)
|
|
|
|
return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply)
|
|
|
|
|
|
def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map, retriever):
|
|
# Lookup all the images and their parent images (if any) inside the manifest.
|
|
# This will let us know which v1 images we need to synthesize and which ones are invalid.
|
|
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
|
|
images_query = lookup_repository_images(repository_id, docker_image_ids)
|
|
image_storage_map = {i.docker_image_id: i.storage for i in images_query}
|
|
|
|
# Rewrite any v1 image IDs that do not match the checksum in the database.
|
|
try:
|
|
rewritten_images = manifest_interface_instance.generate_legacy_layers(image_storage_map,
|
|
retriever)
|
|
rewritten_images = list(rewritten_images)
|
|
parent_image_map = {}
|
|
|
|
for rewritten_image in rewritten_images:
|
|
if not rewritten_image.image_id in image_storage_map:
|
|
parent_image = None
|
|
if rewritten_image.parent_image_id:
|
|
parent_image = parent_image_map.get(rewritten_image.parent_image_id)
|
|
if parent_image is None:
|
|
parent_image = get_image(repository_id, rewritten_image.parent_image_id)
|
|
if parent_image is None:
|
|
return None
|
|
|
|
synthesized = synthesize_v1_image(
|
|
repository_id,
|
|
blob_map[rewritten_image.content_checksum].id,
|
|
blob_map[rewritten_image.content_checksum].image_size,
|
|
rewritten_image.image_id,
|
|
rewritten_image.created,
|
|
rewritten_image.comment,
|
|
rewritten_image.command,
|
|
rewritten_image.compat_json,
|
|
parent_image,
|
|
)
|
|
|
|
parent_image_map[rewritten_image.image_id] = synthesized
|
|
except ManifestException:
|
|
logger.exception("exception when rewriting v1 metadata")
|
|
return None
|
|
|
|
return rewritten_images[-1].image_id
|