135 lines
5.5 KiB
Python
135 lines
5.5 KiB
Python
|
import logging
|
||
|
|
||
|
from peewee import IntegrityError
|
||
|
|
||
|
from data.database import Tag, Manifest, ManifestBlob, ManifestLegacyImage, db_transaction
|
||
|
from data.model.oci.tag import filter_to_alive_tags
|
||
|
from data.model.storage import lookup_repo_storages_by_content_checksum
|
||
|
from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
|
||
|
from image.docker.schema1 import DockerSchema1Manifest, ManifestException
|
||
|
|
||
|
logger = logging.getLogger(__name__)
|
||
|
|
||
|
def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
|
||
|
""" Returns the manifest with the specified digest under the specified repository
|
||
|
or None if none. If allow_dead is True, then manifests referenced by only
|
||
|
dead tags will also be returned.
|
||
|
"""
|
||
|
query = (Manifest
|
||
|
.select()
|
||
|
.where(Manifest.repository == repository_id)
|
||
|
.where(Manifest.digest == manifest_digest))
|
||
|
|
||
|
if not allow_dead:
|
||
|
query = filter_to_alive_tags(query.join(Tag)).group_by(Manifest.id)
|
||
|
|
||
|
try:
|
||
|
return query.get()
|
||
|
except Manifest.DoesNotExist:
|
||
|
return None
|
||
|
|
||
|
|
||
|
def get_or_create_manifest(repository_id, manifest_interface_instance):
|
||
|
""" Returns a tuple of the manifest in the specified repository with the matching digest
|
||
|
(if it already exists) or, if not yet created, creates and returns the manifest, as well as
|
||
|
if the manifest was created. Returns (None, None) if there was an error creating the manifest.
|
||
|
Note that *all* blobs referenced by the manifest must exist already in the repository or this
|
||
|
method will fail with a (None, None).
|
||
|
"""
|
||
|
existing = lookup_manifest(repository_id, manifest_interface_instance.digest, allow_dead=True)
|
||
|
if existing is not None:
|
||
|
return existing, False
|
||
|
|
||
|
assert len(list(manifest_interface_instance.layers)) > 0
|
||
|
|
||
|
# TODO(jschorr): Switch this to supporting schema2 once we're ready.
|
||
|
assert isinstance(manifest_interface_instance, DockerSchema1Manifest)
|
||
|
|
||
|
# Ensure all the blobs in the manifest exist.
|
||
|
digests = manifest_interface_instance.checksums
|
||
|
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
|
||
|
blob_map = {s.content_checksum: s for s in query}
|
||
|
for digest_str in manifest_interface_instance.blob_digests:
|
||
|
if digest_str not in blob_map:
|
||
|
logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
|
||
|
manifest_interface_instance.digest, repository_id)
|
||
|
return None, None
|
||
|
|
||
|
# Determine and populate the legacy image if necessary.
|
||
|
legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map)
|
||
|
if legacy_image_id is None:
|
||
|
return None, None
|
||
|
|
||
|
legacy_image = get_image(repository_id, legacy_image_id)
|
||
|
if legacy_image is None:
|
||
|
return None, None
|
||
|
|
||
|
# Create the manifest and its blobs.
|
||
|
media_type = Manifest.media_type.get_id(manifest_interface_instance.content_type)
|
||
|
storage_ids = {storage.id for storage in blob_map.values()}
|
||
|
|
||
|
with db_transaction():
|
||
|
# Create the manifest.
|
||
|
try:
|
||
|
manifest = Manifest.create(repository=repository_id,
|
||
|
digest=manifest_interface_instance.digest,
|
||
|
media_type=media_type,
|
||
|
manifest_bytes=manifest_interface_instance.bytes)
|
||
|
except IntegrityError:
|
||
|
manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest)
|
||
|
return manifest, False
|
||
|
|
||
|
# Insert the blobs.
|
||
|
blobs_to_insert = [dict(manifest=manifest, repository=repository_id,
|
||
|
blob=storage_id) for storage_id in storage_ids]
|
||
|
if blobs_to_insert:
|
||
|
ManifestBlob.insert_many(blobs_to_insert).execute()
|
||
|
|
||
|
# Set the legacy image (if applicable).
|
||
|
ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)
|
||
|
|
||
|
return manifest, True
|
||
|
|
||
|
|
||
|
def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map):
|
||
|
# Lookup all the images and their parent images (if any) inside the manifest.
|
||
|
# This will let us know which v1 images we need to synthesize and which ones are invalid.
|
||
|
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
|
||
|
images_query = lookup_repository_images(repository_id, docker_image_ids)
|
||
|
image_storage_map = {i.docker_image_id: i.storage for i in images_query}
|
||
|
|
||
|
# Rewrite any v1 image IDs that do not match the checksum in the database.
|
||
|
try:
|
||
|
rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(image_storage_map)
|
||
|
rewritten_images = list(rewritten_images)
|
||
|
parent_image_map = {}
|
||
|
|
||
|
for rewritten_image in rewritten_images:
|
||
|
if not rewritten_image.image_id in image_storage_map:
|
||
|
parent_image = None
|
||
|
if rewritten_image.parent_image_id:
|
||
|
parent_image = parent_image_map.get(rewritten_image.parent_image_id)
|
||
|
if parent_image is None:
|
||
|
parent_image = get_image(repository_id, rewritten_image.parent_image_id)
|
||
|
if parent_image is None:
|
||
|
return None
|
||
|
|
||
|
synthesized = synthesize_v1_image(
|
||
|
repository_id,
|
||
|
blob_map[rewritten_image.content_checksum].id,
|
||
|
blob_map[rewritten_image.content_checksum].image_size,
|
||
|
rewritten_image.image_id,
|
||
|
rewritten_image.created,
|
||
|
rewritten_image.comment,
|
||
|
rewritten_image.command,
|
||
|
rewritten_image.compat_json,
|
||
|
parent_image,
|
||
|
)
|
||
|
|
||
|
parent_image_map[rewritten_image.image_id] = synthesized
|
||
|
except ManifestException:
|
||
|
logger.exception("exception when rewriting v1 metadata")
|
||
|
return None
|
||
|
|
||
|
return rewritten_images[-1].image_id
|