This repository has been archived on 2020-03-24. You can view files and clone it, but cannot push or open issues or pull requests.
quay/data/model/oci/manifest.py

135 lines
5.5 KiB
Python
Raw Normal View History

import logging
from peewee import IntegrityError
from data.database import Tag, Manifest, ManifestBlob, ManifestLegacyImage, db_transaction
from data.model.oci.tag import filter_to_alive_tags
from data.model.storage import lookup_repo_storages_by_content_checksum
from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
from image.docker.schema1 import DockerSchema1Manifest, ManifestException
logger = logging.getLogger(__name__)
def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
""" Returns the manifest with the specified digest under the specified repository
or None if none. If allow_dead is True, then manifests referenced by only
dead tags will also be returned.
"""
query = (Manifest
.select()
.where(Manifest.repository == repository_id)
.where(Manifest.digest == manifest_digest))
if not allow_dead:
query = filter_to_alive_tags(query.join(Tag)).group_by(Manifest.id)
try:
return query.get()
except Manifest.DoesNotExist:
return None
def get_or_create_manifest(repository_id, manifest_interface_instance):
""" Returns a tuple of the manifest in the specified repository with the matching digest
(if it already exists) or, if not yet created, creates and returns the manifest, as well as
if the manifest was created. Returns (None, None) if there was an error creating the manifest.
Note that *all* blobs referenced by the manifest must exist already in the repository or this
method will fail with a (None, None).
"""
existing = lookup_manifest(repository_id, manifest_interface_instance.digest, allow_dead=True)
if existing is not None:
return existing, False
assert len(list(manifest_interface_instance.layers)) > 0
# TODO(jschorr): Switch this to supporting schema2 once we're ready.
assert isinstance(manifest_interface_instance, DockerSchema1Manifest)
# Ensure all the blobs in the manifest exist.
digests = manifest_interface_instance.checksums
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
blob_map = {s.content_checksum: s for s in query}
for digest_str in manifest_interface_instance.blob_digests:
if digest_str not in blob_map:
logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
manifest_interface_instance.digest, repository_id)
return None, None
# Determine and populate the legacy image if necessary.
legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map)
if legacy_image_id is None:
return None, None
legacy_image = get_image(repository_id, legacy_image_id)
if legacy_image is None:
return None, None
# Create the manifest and its blobs.
media_type = Manifest.media_type.get_id(manifest_interface_instance.content_type)
storage_ids = {storage.id for storage in blob_map.values()}
with db_transaction():
# Create the manifest.
try:
manifest = Manifest.create(repository=repository_id,
digest=manifest_interface_instance.digest,
media_type=media_type,
manifest_bytes=manifest_interface_instance.bytes)
except IntegrityError:
manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest)
return manifest, False
# Insert the blobs.
blobs_to_insert = [dict(manifest=manifest, repository=repository_id,
blob=storage_id) for storage_id in storage_ids]
if blobs_to_insert:
ManifestBlob.insert_many(blobs_to_insert).execute()
# Set the legacy image (if applicable).
ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)
return manifest, True
def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map):
# Lookup all the images and their parent images (if any) inside the manifest.
# This will let us know which v1 images we need to synthesize and which ones are invalid.
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
images_query = lookup_repository_images(repository_id, docker_image_ids)
image_storage_map = {i.docker_image_id: i.storage for i in images_query}
# Rewrite any v1 image IDs that do not match the checksum in the database.
try:
rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(image_storage_map)
rewritten_images = list(rewritten_images)
parent_image_map = {}
for rewritten_image in rewritten_images:
if not rewritten_image.image_id in image_storage_map:
parent_image = None
if rewritten_image.parent_image_id:
parent_image = parent_image_map.get(rewritten_image.parent_image_id)
if parent_image is None:
parent_image = get_image(repository_id, rewritten_image.parent_image_id)
if parent_image is None:
return None
synthesized = synthesize_v1_image(
repository_id,
blob_map[rewritten_image.content_checksum].id,
blob_map[rewritten_image.content_checksum].image_size,
rewritten_image.image_id,
rewritten_image.created,
rewritten_image.comment,
rewritten_image.command,
rewritten_image.compat_json,
parent_image,
)
parent_image_map[rewritten_image.image_id] = synthesized
except ManifestException:
logger.exception("exception when rewriting v1 metadata")
return None
return rewritten_images[-1].image_id