quay/endpoints/v2/manifest.py

import logging

from functools import wraps

from flask import make_response, request, url_for

import features

from app import docker_v2_signing_key, app, metric_queue
from auth.registry_jwt_auth import process_registry_jwt_auth
from data import model
from data.types import (
  DockerSchema1Manifest,
  DockerSchema1ManifestBuilder,
  ManifestException,
  DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE,
  DOCKER_SCHEMA2_CONTENT_TYPES,
)
from digest import digest_tools
from endpoints.common import parse_repository_name
from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
                                 NameInvalid)
from endpoints.trackhelper import track_and_log
from endpoints.notificationhelper import spawn_notification
from util.registry.replication import queue_storage_replication
from util.names import VALID_TAG_PATTERN

logger = logging.getLogger(__name__)


BASE_MANIFEST_ROUTE = '/<repopath:repository>/manifests/<regex("{0}"):manifest_ref>'
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)

@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull'])
@require_repo_read
@anon_protect
def fetch_manifest_by_tagname(namespace_name, repo_name, tag_name):
  manifest = v2.get_manifest_by_tag(namespace_name, repo_name, tag_name)
  if manifest is None:
    tag = v2.get_active_tag(namespace_name, repo_name, tag_name)
    if tag is None:
      raise ManifestUnknown()

    manifest = _generate_and_store_manifest(namespace_name, repo_name, tag_name)
    if manifest is None:
      raise ManifestUnknown()

  repo = v2.get_repository(namespace_name, repo_name)
  if repo is not None:
    track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01)
    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])

  response = make_response(manifest.bytes, 200)
  response.headers['Content-Type'] = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
  response.headers['Docker-Content-Digest'] = manifest.digest
  return response


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull'])
@require_repo_read
@anon_protect
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
  manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, manifest_ref)
  if manifest is None:
    # Without a tag name to reference, we can't make an attempt to generate the manifest
    raise ManifestUnknown()

  repo = v2.get_repository(namespace_name, repo_name)
  if repo is not None:
    track_and_log('pull_repo', repo)
    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])

  response = make_response(manifest.json, 200)
  response.headers['Content-Type'] = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
  response.headers['Docker-Content-Digest'] = manifest.digest
  return response


def _reject_manifest2_schema2(func):
  @wraps(func)
  def wrapped(*args, **kwargs):
    if request.content_type in DOCKER_SCHEMA2_CONTENT_TYPES:
      raise ManifestInvalid(detail={'message': 'manifest schema version not supported'},
                            http_status_code=415)
    return func(*args, **kwargs)
  return wrapped


@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
@_reject_manifest2_schema2
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull', 'push'])
@require_repo_write
@anon_protect
def write_manifest_by_tagname(namespace_name, repo_name, tag_name):
  try:
    manifest = DockerSchema1Manifest(request.data)
  except ManifestException as me:
    raise ManifestInvalid(detail={'message': me.message})

  if manifest.tag != tag_name:
    raise TagInvalid()

  return _write_manifest(namespace_name, repo_name, manifest)


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
@_reject_manifest2_schema2
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull', 'push'])
@require_repo_write
@anon_protect
def write_manifest_by_digest(namespace_name, repo_name, digest):
  try:
    manifest = DockerSchema1Manifest(request.data)
  except ManifestException as me:
    raise ManifestInvalid(detail={'message': me.message})

  if manifest.digest != digest:
    raise ManifestInvalid(detail={'message': 'manifest digest mismatch'})

  return _write_manifest(namespace_name, repo_name, manifest)


def _write_manifest(namespace_name, repo_name, manifest):
  if (manifest.namespace == '' and
      features.LIBRARY_SUPPORT and
      namespace_name == app.config['LIBRARY_NAMESPACE']):
    pass
  elif manifest.namespace != namespace_name:
    raise NameInvalid()

  if manifest.repo_name != repo_name:
    raise NameInvalid()

  # Ensure that the repository exists.
  repo = v2.get_repository(namespace_name, repo_name)
  if repo is None:
    raise NameInvalid()

  if not manifest.layers:
    raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})

  # Ensure all the blobs in the manifest exist.
  storage_query = model.storage.lookup_repo_storages_by_content_checksum(repo, manifest.checksums)
  storage_map = {storage.content_checksum: storage for storage in storage_query}
  for extracted_layer_metadata in manifest.layers:
    digest_str = str(extracted_layer_metadata.digest)
    if digest_str not in storage_map:
      raise BlobUnknown(detail={'digest': digest_str})

  # Lookup all the images and their parent images (if any) inside the manifest.
  # This will let us know which v1 images we need to synthesize and which ones are invalid.
  all_image_ids = list(manifest.docker_image_ids | manifest.parent_image_ids)
  images = v2.docker_v1_metadata_by_image_id(namespace_name, repo_name, all_image_ids)
  images_map = {image.image_id: image for image in images}

  # Rewrite any v1 image IDs that do not match the checksum in the database.
  try:
    rewritten_images = manifest.rewrite_invalid_image_ids(images_map)
    for rewritten_image in rewritten_images:
      image = v2.synthesize_v1_image(
        repo,
        storage_map[rewritten_image.content_checksum],
        rewritten_image.image_id,
        rewritten_image.created,
        rewritten_image.comment,
        rewritten_image.command,
        rewritten_image.compat_json,
        rewritten_image.parent_image_id,
      )
      images_map[image.image_id] = image
  except ManifestException as me:
    raise ManifestInvalid(detail={'message': me.message})

  # Store the manifest pointing to the tag.
  leaf_layer_id = images_map[manifest.layers[-1].v1_metadata.image_id].image_id
  v2.save_manifest(namespace_name, repo_name, tag_name, leaf_layer_id, manifest.digest, manifest.bytes)

  # Queue all blob manifests for replication.
  # TODO(jschorr): Find a way to optimize this insertion.
  if features.STORAGE_REPLICATION:
    for extracted_v1_metadata in manifest.layers:
      digest_str = str(extracted_v1_metadata.digest)
      queue_storage_replication(namespace_name, storage_map[digest_str])

  track_and_log('push_repo', repo, tag=manifest.tag)
  spawn_notification(repo, 'repo_push', {'updated_tags': [manifest.tag]})
  metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2'])

  response = make_response('OK', 202)
  response.headers['Docker-Content-Digest'] = manifest.digest
  response.headers['Location'] = url_for('v2.fetch_manifest_by_digest',
                                         repository='%s/%s' % (namespace_name, repo_name),
                                         manifest_ref=manifest.digest)
  return response


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
@parse_repository_name()
@process_registry_jwt_auth(scopes=['pull', 'push'])
@require_repo_write
@anon_protect
def delete_manifest_by_digest(namespace_name, repo_name, digest):
  """
  Delete the manifest specified by the digest.

  Note: there is no equivalent method for deleting by tag name because it is
  forbidden by the spec.
  """
  tag = v2.get_tag_by_manifest_digest(namespace_name, repo_name, digest)
  if tag is None:
    # TODO(jzelinskie): disambiguate between no manifest and no tag
    raise ManifestUnknown()

  # Mark the tag as no longer alive.
  deleted = v2.delete_tag(namespace_name, repo_name, tag.name)
  if not deleted:
    # Tag was not alive.
    raise ManifestUnknown()

  track_and_log('delete_tag', tag.repository, tag=tag.name, digest=digest)

  return make_response('', 202)


def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
  # Find the v1 metadata for this image and its parents.
  v1_metadata = v2.docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name)
  parents_v1_metadata = v2.get_parents_docker_v1_metadata(namespace_name, repo_name,
                                                          v1_metadata.image_id)

  # If the manifest is being generated under the library namespace, then we make its namespace
  # empty.
  manifest_namespace = namespace_name
  if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
    manifest_namespace = ''

  # Create and populate the manifest builder
  builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)

  # Add the leaf layer
  builder.add_layer(v1_metadata.content_checksum, v1_metadata.compat_json)

  for parent_v1_metadata in parents_v1_metadata:
    builder.add_layer(parent_v1_metadata.content_checksum, parent_v1_metadata.compat_json)

  # Sign the manifest with our signing key.
  manifest = builder.build(docker_v2_signing_key)

  # Write the manifest to the DB.
  v2.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest,
                                    manifest.bytes)
  return manifest