quay/endpoints/v2/manifest.py

# XXX This code is not yet ready to be run in production, and should remain disabled until such
# XXX time as this notice is removed.

import logging
import re
import jwt.utils
import json

from flask import make_response, request, url_for
from collections import namedtuple, OrderedDict
from jwkest.jws import SIGNER_ALGS
from datetime import datetime

from app import storage, docker_v2_signing_key
from auth.jwt_auth import process_jwt_auth
from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnverified,
                                 ManifestUnknown, TagInvalid, NameInvalid)
from endpoints.trackhelper import track_and_log
from endpoints.notificationhelper import spawn_notification
from digest import digest_tools
from data import model


logger = logging.getLogger(__name__)


VALID_TAG_PATTERN = r'[\w][\w.-]{0,127}'

BASE_MANIFEST_ROUTE = '/<namespace>/<repo_name>/manifests/<regex("{0}"):manifest_ref>'
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)


ISO_DATETIME_FORMAT_ZULU = '%Y-%m-%dT%H:%M:%SZ'
JWS_ALGORITHM = 'RS256'


ImageMetadata = namedtuple('ImageMetadata', ['digest', 'v1_metadata', 'v1_metadata_str'])
ExtractedV1Metadata = namedtuple('ExtractedV1Metadata', ['docker_id', 'parent', 'created',
                                                         'comment', 'command'])


_SIGNATURES_KEY = 'signatures'
_PROTECTED_KEY = 'protected'
_FORMAT_LENGTH_KEY = 'formatLength'
_FORMAT_TAIL_KEY = 'formatTail'
_REPO_NAME_KEY = 'name'
_REPO_TAG_KEY = 'tag'
_FS_LAYERS_KEY = 'fsLayers'
_HISTORY_KEY = 'history'
_BLOB_SUM_KEY = 'blobSum'
_V1_COMPAT_KEY = 'v1Compatibility'
_ARCH_KEY = 'architecture'
_SCHEMA_VER = 'schemaVersion'


class SignedManifest(object):

  def __init__(self, manifest_bytes):
    self._bytes = manifest_bytes

    self._parsed = json.loads(manifest_bytes)
    self._signatures = self._parsed[_SIGNATURES_KEY]
    self._namespace, self._repo_name = self._parsed[_REPO_NAME_KEY].split('/')
    self._tag = self._parsed[_REPO_TAG_KEY]

    self._validate()

  def _validate(self):
    pass

  @property
  def signatures(self):
    return self._signatures

  @property
  def namespace(self):
    return self._namespace

  @property
  def repo_name(self):
    return self._repo_name

  @property
  def tag(self):
    return self._tag

  @property
  def bytes(self):
    return self._bytes

  @property
  def digest(self):
    return digest_tools.sha256_digest(self.payload)

  @property
  def layers(self):
    """ Returns a generator of objects that have the blobSum and v1Compatibility keys in them,
        starting from the leaf image and working toward the base node.
    """
    for blob_sum_obj, history_obj in reversed(zip(self._parsed[_FS_LAYERS_KEY],
                                                  self._parsed[_HISTORY_KEY])):

      try:
        image_digest = digest_tools.Digest.parse_digest(blob_sum_obj[_BLOB_SUM_KEY])
      except digest_tools.InvalidDigestException:
        raise ManifestInvalid()

      metadata_string = history_obj[_V1_COMPAT_KEY]

      v1_metadata = json.loads(metadata_string)
      command_list = v1_metadata.get('container_config', {}).get('Cmd', None)
      command = json.dumps(command_list) if command_list else None

      extracted = ExtractedV1Metadata(v1_metadata['id'], v1_metadata.get('parent'),
                                      v1_metadata.get('created'), v1_metadata.get('comment'),
                                      command)
      yield ImageMetadata(image_digest, extracted, metadata_string)

  @property
  def payload(self):
    protected = str(self._signatures[0][_PROTECTED_KEY])

    parsed_protected = json.loads(jwt.utils.base64url_decode(protected))
    logger.debug('parsed_protected: %s', parsed_protected)

    signed_content_head = self._bytes[:parsed_protected[_FORMAT_LENGTH_KEY]]
    logger.debug('signed content head: %s', signed_content_head)

    signed_content_tail = jwt.utils.base64url_decode(str(parsed_protected[_FORMAT_TAIL_KEY]))
    logger.debug('signed content tail: %s', signed_content_tail)
    return signed_content_head + signed_content_tail


class SignedManifestBuilder(object):
  """ Class which represents a manifest which is currently being built.
  """
  def __init__(self, namespace, repo_name, tag, architecture='amd64', schema_ver=1):
    self._base_payload = {
      _REPO_TAG_KEY: tag,
      _REPO_NAME_KEY: '{0}/{1}'.format(namespace, repo_name),
      _ARCH_KEY: architecture,
      _SCHEMA_VER: schema_ver,
    }

    self._fs_layer_digests = []
    self._history = []

  def add_layer(self, layer_digest, v1_json_metadata):
    self._fs_layer_digests.append({
      _BLOB_SUM_KEY: layer_digest,
    })
    self._history.append({
      _V1_COMPAT_KEY: v1_json_metadata,
    })

  def build(self, json_web_key):
    """ Build the payload and sign it, returning a SignedManifest object.
    """
    payload = OrderedDict(self._base_payload)
    payload.update({
      _HISTORY_KEY: self._history,
      _FS_LAYERS_KEY: self._fs_layer_digests,
    })

    payload_str = json.dumps(payload, indent=3)

    split_point = payload_str.rfind('\n}')

    protected_payload = {
      'formatTail': jwt.utils.base64url_encode(payload_str[split_point:]),
      'formatLength': split_point,
      'time': datetime.utcnow().strftime(ISO_DATETIME_FORMAT_ZULU),
    }
    protected = jwt.utils.base64url_encode(json.dumps(protected_payload))
    logger.debug('Generated protected block: %s', protected)

    bytes_to_sign = '{0}.{1}'.format(protected, jwt.utils.base64url_encode(payload_str))

    signer = SIGNER_ALGS[JWS_ALGORITHM]
    signature = jwt.utils.base64url_encode(signer.sign(bytes_to_sign, json_web_key.get_key()))
    logger.debug('Generated signature: %s', signature)

    signature_block = {
      'header': {
        'jwk': json_web_key.to_dict(),
        'alg': JWS_ALGORITHM,
      },
      'signature': signature,
      _PROTECTED_KEY: protected,
    }

    logger.debug('Encoded signature block: %s', json.dumps(signature_block))

    payload.update({
      _SIGNATURES_KEY: [signature_block],
    })

    return SignedManifest(json.dumps(payload, indent=3))


@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
@process_jwt_auth
@require_repo_read
@anon_protect
def fetch_manifest_by_tagname(namespace, repo_name, manifest_ref):
  try:
    manifest = model.tag.load_tag_manifest(namespace, repo_name, manifest_ref)
  except model.InvalidManifestException:
    try:
      manifest = _generate_and_store_manifest(namespace, repo_name, manifest_ref)
    except model.DataModelException:
      logger.exception('Exception when generating manifest for %s/%s:%s', namespace, repo_name,
                       manifest_ref)
      raise ManifestUnknown()

  return make_response(manifest.json_data, 200)


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
@process_jwt_auth
@require_repo_read
@anon_protect
def fetch_manifest_by_digest(namespace, repo_name, manifest_ref):
  try:
    manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
  except model.InvalidManifestException:
    # Without a tag name to reference, we can't make an attempt to generate the manifest
    raise ManifestUnknown()

  return make_response(manifest.json_data, 200)


@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
@process_jwt_auth
@require_repo_write
@anon_protect
def write_manifest_by_tagname(namespace, repo_name, manifest_ref):
  manifest = SignedManifest(request.data)
  if manifest.tag != manifest_ref:
    raise TagInvalid()

  return _write_manifest(namespace, repo_name, manifest)


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
@process_jwt_auth
@require_repo_write
@anon_protect
def write_manifest_by_digest(namespace, repo_name, manifest_ref):
  manifest = SignedManifest(request.data)
  if manifest.digest != manifest_ref:
    raise ManifestInvalid()

  return _write_manifest(namespace, repo_name, manifest)


def _write_manifest(namespace, repo_name, manifest):
  # Ensure that the manifest is for this repository.
  if manifest.namespace != namespace or manifest.repo_name != repo_name:
    raise NameInvalid()

  # Ensure that the repository exists.
  repo = model.repository.get_repository(namespace, repo_name)
  if repo is None:
    raise NameInvalid()

  # Lookup all the images and their parent images (if any) inside the manifest. This will let us
  # know which V1 images we need to synthesize and which ones are invalid.
  layers = list(manifest.layers)

  docker_image_ids = [mdata.v1_metadata.docker_id for mdata in layers]
  parent_image_ids = [mdata.v1_metadata.parent for mdata in layers
                      if mdata.v1_metadata.parent]
  all_image_ids = list(set(docker_image_ids + parent_image_ids))

  images_query = model.image.lookup_repository_images(repo, all_image_ids)
  images_map = {image.docker_image_id: image for image in images_query}

  # Lookup the storages associated with each blob in the manifest.
  checksums = [str(mdata.digest) for mdata in manifest.layers]
  storage_query = model.storage.lookup_repo_storages_by_checksum(repo, checksums)
  storage_map = {storage.checksum: storage for storage in storage_query}

  # Synthesize the V1 metadata for each layer.
  manifest_digest = manifest.digest
  tag_name = manifest.tag

  for mdata in layers:
    digest_str = str(mdata.digest)
    v1_mdata = mdata.v1_metadata

    # If there is already a V1 image for this layer, nothing more to do.
    if v1_mdata.docker_id in images_map:
      continue

    # Lookup the parent image for the layer, if any.
    parent_image = None
    if v1_mdata.parent is not None:
      parent_image = images_map.get(v1_mdata.parent)
      if parent_image is None:
        msg = 'Parent not found with docker image id {0}'.format(v1_mdata.parent)
        raise ManifestInvalid(detail={'message': msg})

    # Synthesize and store the v1 metadata in the db.
    blob_storage = storage_map.get(digest_str)
    if blob_storage is None:
      raise BlobUnknown(detail={'digest': digest_str})

    image = model.image.synthesize_v1_image(repo, blob_storage, v1_mdata.docker_id,
                                            v1_mdata.created, v1_mdata.comment, v1_mdata.command,
                                            mdata.v1_metadata_str, parent_image)

    images_map[v1_mdata.docker_id] = image

  if not layers:
    # The manifest doesn't actually reference any layers!
    raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})

  # Store the manifest pointing to the tag.
  leaf_layer = layers[0]
  model.tag.store_tag_manifest(namespace, repo_name, tag_name, leaf_layer.v1_metadata.docker_id,
                               manifest_digest, request.data)

  # Spawn the repo_push event.
  event_data = {
    'updated_tags': [tag_name],
  }

  track_and_log('push_repo', repo)
  spawn_notification(repo, 'repo_push', event_data)

  response = make_response('OK', 202)
  response.headers['Docker-Content-Digest'] = manifest_digest
  response.headers['Location'] = url_for('v2.fetch_manifest_by_digest', namespace=namespace,
                                         repo_name=repo_name, manifest_ref=manifest_digest)
  return response


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
@process_jwt_auth
@require_repo_write
@anon_protect
def delete_manifest_by_digest(namespace, repo_name, manifest_ref):
  """ Delete the manifest specified by the digest. Note: there is no equivalent
      method for deleting by tag name because it is forbidden by the spec.
  """
  try:
    manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
  except model.InvalidManifestException:
    # Without a tag name to reference, we can't make an attempt to generate the manifest
    raise ManifestUnknown()

  manifest.delete_instance()

  return make_response('', 202)


def _generate_and_store_manifest(namespace, repo_name, tag_name):
  # First look up the tag object and its ancestors
  image = model.tag.get_tag_image(namespace, repo_name, tag_name)
  parents = model.image.get_parent_images(namespace, repo_name, image)

  # Create and populate the manifest builder
  builder = SignedManifestBuilder(namespace, repo_name, tag_name)

  # Add the leaf layer
  builder.add_layer(image.storage.checksum, __get_and_backfill_image_metadata(image))

  for parent in parents:
    builder.add_layer(parent.storage.checksum, __get_and_backfill_image_metadata(parent))

  # Sign the manifest with our signing key.
  manifest = builder.build(docker_v2_signing_key)
  manifest_row = model.tag.associate_generated_tag_manifest(namespace, repo_name, tag_name,
                                                            manifest.digest, manifest.bytes)

  return manifest_row


def __get_and_backfill_image_metadata(image):
  image_metadata = image.v1_json_metadata
  if image_metadata is None:
    logger.warning('Loading metadata from storage for image id: %s', image.id)

    image.v1_json_metadata = model.image.get_image_json(image)
    logger.info('Saving backfilled metadata for image id: %s', image.id)
    image.save()

  return image_metadata