quay/data/model/oci/manifest.py

import logging

from peewee import IntegrityError

from data.database import Tag, Manifest, ManifestBlob, ManifestLegacyImage, db_transaction
from data.model.oci.tag import filter_to_alive_tags
from data.model.storage import lookup_repo_storages_by_content_checksum
from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
from image.docker.schema1 import DockerSchema1Manifest, ManifestException

logger = logging.getLogger(__name__)

def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
  """ Returns the manifest with the specified digest under the specified repository
      or None if none. If allow_dead is True, then manifests referenced by only
      dead tags will also be returned.
  """
  query = (Manifest
           .select()
           .where(Manifest.repository == repository_id)
           .where(Manifest.digest == manifest_digest))

  if not allow_dead:
    query = filter_to_alive_tags(query.join(Tag)).group_by(Manifest.id)

  try:
    return query.get()
  except Manifest.DoesNotExist:
    return None


def get_or_create_manifest(repository_id, manifest_interface_instance):
  """ Returns a tuple of the manifest in the specified repository with the matching digest
      (if it already exists) or, if not yet created, creates and returns the manifest, as well as
      if the manifest was created. Returns (None, None) if there was an error creating the manifest.
      Note that *all* blobs referenced by the manifest must exist already in the repository or this
      method will fail with a (None, None).
  """
  existing = lookup_manifest(repository_id, manifest_interface_instance.digest, allow_dead=True)
  if existing is not None:
    return existing, False

  assert len(list(manifest_interface_instance.layers)) > 0

  # TODO(jschorr): Switch this to supporting schema2 once we're ready.
  assert isinstance(manifest_interface_instance, DockerSchema1Manifest)

  # Ensure all the blobs in the manifest exist.
  digests = manifest_interface_instance.checksums
  query = lookup_repo_storages_by_content_checksum(repository_id, digests)
  blob_map = {s.content_checksum: s for s in query}
  for digest_str in manifest_interface_instance.blob_digests:
    if digest_str not in blob_map:
      logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
                     manifest_interface_instance.digest, repository_id)
      return None, None

  # Determine and populate the legacy image if necessary.
  legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map)
  if legacy_image_id is None:
    return None, None

  legacy_image = get_image(repository_id, legacy_image_id)
  if legacy_image is None:
    return None, None

  # Create the manifest and its blobs.
  media_type = Manifest.media_type.get_id(manifest_interface_instance.content_type)
  storage_ids = {storage.id for storage in blob_map.values()}

  with db_transaction():
    # Create the manifest.
    try:
      manifest = Manifest.create(repository=repository_id,
                                 digest=manifest_interface_instance.digest,
                                 media_type=media_type,
                                 manifest_bytes=manifest_interface_instance.bytes)
    except IntegrityError:
      manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest)
      return manifest, False

    # Insert the blobs.
    blobs_to_insert = [dict(manifest=manifest, repository=repository_id,
                            blob=storage_id) for storage_id in storage_ids]
    if blobs_to_insert:
      ManifestBlob.insert_many(blobs_to_insert).execute()

    # Set the legacy image (if applicable).
    ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)

    return manifest, True


def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map):
  # Lookup all the images and their parent images (if any) inside the manifest.
  # This will let us know which v1 images we need to synthesize and which ones are invalid.
  docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
  images_query = lookup_repository_images(repository_id, docker_image_ids)
  image_storage_map = {i.docker_image_id: i.storage for i in images_query}

  # Rewrite any v1 image IDs that do not match the checksum in the database.
  try:
    rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(image_storage_map)
    rewritten_images = list(rewritten_images)
    parent_image_map = {}

    for rewritten_image in rewritten_images:
      if not rewritten_image.image_id in image_storage_map:
        parent_image = None
        if rewritten_image.parent_image_id:
          parent_image = parent_image_map.get(rewritten_image.parent_image_id)
          if parent_image is None:
            parent_image = get_image(repository_id, rewritten_image.parent_image_id)
            if parent_image is None:
              return None

        synthesized = synthesize_v1_image(
          repository_id,
          blob_map[rewritten_image.content_checksum].id,
          blob_map[rewritten_image.content_checksum].image_size,
          rewritten_image.image_id,
          rewritten_image.created,
          rewritten_image.comment,
          rewritten_image.command,
          rewritten_image.compat_json,
          parent_image,
        )

        parent_image_map[rewritten_image.image_id] = synthesized
  except ManifestException:
    logger.exception("exception when rewriting v1 metadata")
    return None

  return rewritten_images[-1].image_id