quay/endpoints/v2/manifest.py

import logging

from functools import wraps

from flask import request, url_for, Response

import features

from app import docker_v2_signing_key, app, metric_queue
from auth.registry_jwt_auth import process_registry_jwt_auth
from digest import digest_tools
from endpoints.decorators import anon_protect, parse_repository_name
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
from endpoints.v2.models_interface import Label
from endpoints.v2.models_pre_oci import data_model as model
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
                                 NameInvalid, TagExpired)
from endpoints.v2.labelhandlers import handle_label
from image.docker import ManifestException
from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES, OCI_CONTENT_TYPES
from notifications import spawn_notification
from util.audit import track_and_log
from util.names import VALID_TAG_PATTERN
from util.registry.replication import queue_replication_batch
from util.validation import is_json

logger = logging.getLogger(__name__)

BASE_MANIFEST_ROUTE = '/<repopath:repository>/manifests/<regex("{0}"):manifest_ref>'
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)


@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull'])
@require_repo_read
@anon_protect
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
  manifest = model.get_manifest_by_tag(namespace_name, repo_name, manifest_ref)
  if manifest is None:
    has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref)
    if not has_tag:
      has_expired_tag = model.has_tag(namespace_name, repo_name, manifest_ref)
      if has_expired_tag:
        logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name,
                     repo_name)
        msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref
        raise TagExpired(msg)
      else:
        raise ManifestUnknown()

    manifest = _generate_and_store_manifest(namespace_name, repo_name, manifest_ref)
    if manifest is None:
      raise ManifestUnknown()

  repo = model.get_repository(namespace_name, repo_name)
  if repo is not None:
    track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01,
                  tag=manifest_ref)
    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])

  return Response(
    manifest.json,
    status=200,
    headers={'Content-Type': manifest.media_type,
             'Docker-Content-Digest': manifest.digest},)


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull'])
@require_repo_read
@anon_protect
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
  manifest = model.get_manifest_by_digest(namespace_name, repo_name, manifest_ref)
  if manifest is None:
    # Without a tag name to reference, we can't make an attempt to generate the manifest
    raise ManifestUnknown()

  repo = model.get_repository(namespace_name, repo_name)
  if repo is not None:
    track_and_log('pull_repo', repo, manifest_digest=manifest_ref)
    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])

  return Response(manifest.json, status=200, headers={
    'Content-Type': manifest.media_type,
    'Docker-Content-Digest': manifest.digest})


def _reject_manifest2_schema2(func):
  @wraps(func)
  def wrapped(*args, **kwargs):
    if request.content_type in (DOCKER_SCHEMA2_CONTENT_TYPES | OCI_CONTENT_TYPES):
      raise ManifestInvalid(detail={'message': 'manifest schema version not supported'},
                            http_status_code=415)
    return func(*args, **kwargs)

  return wrapped


@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
@_reject_manifest2_schema2
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull', 'push'])
@require_repo_write
@anon_protect
def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
  try:
    manifest = DockerSchema1Manifest(request.data)
  except ManifestException as me:
    logger.exception("failed to parse manifest when writing by tagname")
    raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})

  if manifest.tag != manifest_ref:
    raise TagInvalid()

  return _write_manifest_and_log(namespace_name, repo_name, manifest)


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
@_reject_manifest2_schema2
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull', 'push'])
@require_repo_write
@anon_protect
def write_manifest_by_digest(namespace_name, repo_name, manifest_ref):
  try:
    manifest = DockerSchema1Manifest(request.data)
  except ManifestException as me:
    logger.exception("failed to parse manifest when writing by digest")
    raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})

  if manifest.digest != manifest_ref:
    raise ManifestInvalid(detail={'message': 'manifest digest mismatch'})

  return _write_manifest_and_log(namespace_name, repo_name, manifest)


def _write_manifest(namespace_name, repo_name, manifest):
  if (manifest.namespace == '' and features.LIBRARY_SUPPORT and
      namespace_name == app.config['LIBRARY_NAMESPACE']):
    pass
  elif manifest.namespace != namespace_name:
    raise NameInvalid()

  if manifest.repo_name != repo_name:
    raise NameInvalid()

  # Ensure that the repository exists.
  repo = model.get_repository(namespace_name, repo_name)
  if repo is None:
    raise NameInvalid()

  if not manifest.layers:
    raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})

  # Ensure all the blobs in the manifest exist.
  storage_map = model.lookup_blobs_by_digest(repo, manifest.checksums)
  for layer in manifest.layers:
    digest_str = str(layer.digest)
    if digest_str not in storage_map:
      raise BlobUnknown(detail={'digest': digest_str})

  # Lookup all the images and their parent images (if any) inside the manifest.
  # This will let us know which v1 images we need to synthesize and which ones are invalid.
  all_image_ids = list(manifest.parent_image_ids | manifest.image_ids)
  images_map = model.get_docker_v1_metadata_by_image_id(repo, all_image_ids)

  # Rewrite any v1 image IDs that do not match the checksum in the database.
  try:
    # TODO: make this batch and read the parent image from the previous iteration, rather than
    # reloading it.
    rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map))
    for rewritten_image in rewritten_images:
      if not rewritten_image.image_id in images_map:
        model.synthesize_v1_image(
          repo,
          storage_map[rewritten_image.content_checksum],
          rewritten_image.image_id,
          rewritten_image.created,
          rewritten_image.comment,
          rewritten_image.command,
          rewritten_image.compat_json,
          rewritten_image.parent_image_id,
        )
  except ManifestException as me:
    logger.exception("exception when rewriting v1 metadata")
    raise ManifestInvalid(detail={'message': 'failed synthesizing v1 metadata: %s' % me.message})

  # Store the manifest pointing to the tag.
  leaf_layer_id = rewritten_images[-1].image_id
  newly_created = model.save_manifest(repo, manifest.tag, leaf_layer_id, manifest.digest,
                                      manifest.bytes)
  if newly_created:
    # TODO: make this batch
    labels = []
    for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
      media_type = 'application/json' if is_json(value) else 'text/plain'
      labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type))
      handle_label(key, value, namespace_name, repo_name, manifest.digest)

    model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels)

  return repo, storage_map


def _write_manifest_and_log(namespace_name, repo_name, manifest):
  repo, storage_map = _write_manifest(namespace_name, repo_name, manifest)

  # Queue all blob manifests for replication.
  if features.STORAGE_REPLICATION:
    with queue_replication_batch(namespace_name) as queue_storage_replication:
      for layer in manifest.layers:
        digest_str = str(layer.digest)
        queue_storage_replication(storage_map[digest_str])

  track_and_log('push_repo', repo, tag=manifest.tag)
  spawn_notification(repo, 'repo_push', {'updated_tags': [manifest.tag]})
  metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])

  return Response(
    'OK',
    status=202,
    headers={
      'Docker-Content-Digest': manifest.digest,
      'Location':
        url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name),
                manifest_ref=manifest.digest),
    },
  )


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull', 'push'])
@require_repo_write
@anon_protect
def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
  """
  Delete the manifest specified by the digest.

  Note: there is no equivalent method for deleting by tag name because it is
  forbidden by the spec.
  """
  tags = model.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref)
  if not tags:
    raise ManifestUnknown()

  for tag in tags:
    track_and_log('delete_tag', tag.repository, tag=tag.name, digest=manifest_ref)

  return Response(status=202)


def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
  # Find the v1 metadata for this image and its parents.
  v1_metadata = model.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name)
  parents_v1_metadata = model.get_parents_docker_v1_metadata(namespace_name, repo_name,
                                                             v1_metadata.image_id)

  # If the manifest is being generated under the library namespace, then we make its namespace
  # empty.
  manifest_namespace = namespace_name
  if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
    manifest_namespace = ''

  # Create and populate the manifest builder
  builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)

  # Add the leaf layer
  builder.add_layer(v1_metadata.content_checksum, v1_metadata.compat_json)

  for parent_v1_metadata in parents_v1_metadata:
    builder.add_layer(parent_v1_metadata.content_checksum, parent_v1_metadata.compat_json)

  # Sign the manifest with our signing key.
  manifest = builder.build(docker_v2_signing_key)

  # Write the manifest to the DB.
  model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest,
                                       manifest.bytes)
  return manifest