quay/endpoints/v2/manifest.py

# XXX This code is not yet ready to be run in production, and should remain disabled until such
# XXX time as this notice is removed.

import logging
import re
import jwt.utils
import json

from flask import make_response, request, url_for
from collections import namedtuple, OrderedDict
from jwkest.jws import SIGNER_ALGS
from datetime import datetime

from app import storage, docker_v2_signing_key
from auth.jwt_auth import process_jwt_auth
from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnverified,
                                 ManifestUnknown, TagInvalid, NameInvalid)
from endpoints.trackhelper import track_and_log
from endpoints.notificationhelper import spawn_notification
from digest import digest_tools
from data import model


logger = logging.getLogger(__name__)


VALID_TAG_PATTERN = r'[\w][\w.-]{0,127}'

BASE_MANIFEST_ROUTE = '/<namespace>/<repo_name>/manifests/<regex("{0}"):manifest_ref>'
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)


ISO_DATETIME_FORMAT_ZULU = '%Y-%m-%dT%H:%M:%SZ'
JWS_ALGORITHM = 'RS256'


ImageMetadata = namedtuple('ImageMetadata', ['digest', 'v1_metadata', 'v1_metadata_str'])
ExtractedV1Metadata = namedtuple('ExtractedV1Metadata', ['docker_id', 'parent', 'created',
                                                         'comment', 'command'])


_SIGNATURES_KEY = 'signatures'
_PROTECTED_KEY = 'protected'
_FORMAT_LENGTH_KEY = 'formatLength'
_FORMAT_TAIL_KEY = 'formatTail'
_REPO_NAME_KEY = 'name'
_REPO_TAG_KEY = 'tag'
_FS_LAYERS_KEY = 'fsLayers'
_HISTORY_KEY = 'history'
_BLOB_SUM_KEY = 'blobSum'
_V1_COMPAT_KEY = 'v1Compatibility'
_ARCH_KEY = 'architecture'
_SCHEMA_VER = 'schemaVersion'


class SignedManifest(object):

  def __init__(self, manifest_bytes):
    self._bytes = manifest_bytes

    self._parsed = json.loads(manifest_bytes)
    self._signatures = self._parsed[_SIGNATURES_KEY]
    self._namespace, self._repo_name = self._parsed[_REPO_NAME_KEY].split('/')
    self._tag = self._parsed[_REPO_TAG_KEY]

    self._validate()

  def _validate(self):
    pass

  @property
  def signatures(self):
    return self._signatures

  @property
  def namespace(self):
    return self._namespace

  @property
  def repo_name(self):
    return self._repo_name

  @property
  def tag(self):
    return self._tag

  @property
  def bytes(self):
    return self._bytes

  @property
  def digest(self):
    return digest_tools.sha256_digest(self.payload)

  @property
  def layers(self):
    """ Returns a generator of objects that have the blobSum and v1Compatibility keys in them,
        starting from the root image and working toward the leaf node.
    """
    for blob_sum_obj, history_obj in reversed(zip(self._parsed[_FS_LAYERS_KEY],
                                                  self._parsed[_HISTORY_KEY])):

      try:
        image_digest = digest_tools.Digest.parse_digest(blob_sum_obj[_BLOB_SUM_KEY])
      except digest_tools.InvalidDigestException:
        raise ManifestInvalid()

      metadata_string = history_obj[_V1_COMPAT_KEY]

      v1_metadata = json.loads(metadata_string)
      command_list = v1_metadata.get('container_config', {}).get('Cmd', None)
      command = json.dumps(command_list) if command_list else None

      extracted = ExtractedV1Metadata(v1_metadata['id'], v1_metadata.get('parent'),
                                      v1_metadata.get('created'), v1_metadata.get('comment'),
                                      command)
      yield ImageMetadata(image_digest, extracted, metadata_string)

  @property
  def payload(self):
    protected = str(self._signatures[0][_PROTECTED_KEY])

    parsed_protected = json.loads(jwt.utils.base64url_decode(protected))
    logger.debug('parsed_protected: %s', parsed_protected)

    signed_content_head = self._bytes[:parsed_protected[_FORMAT_LENGTH_KEY]]
    logger.debug('signed content head: %s', signed_content_head)

    signed_content_tail = jwt.utils.base64url_decode(str(parsed_protected[_FORMAT_TAIL_KEY]))
    logger.debug('signed content tail: %s', signed_content_tail)
    return signed_content_head + signed_content_tail


class SignedManifestBuilder(object):
  """ Class which represents a manifest which is currently being built.
  """
  def __init__(self, namespace, repo_name, tag, architecture='amd64', schema_ver=1):
    self._base_payload = {
      _REPO_TAG_KEY: tag,
      _REPO_NAME_KEY: '{0}/{1}'.format(namespace, repo_name),
      _ARCH_KEY: architecture,
      _SCHEMA_VER: schema_ver,
    }

    self._fs_layer_digests = []
    self._history = []

  def add_layer(self, layer_digest, v1_json_metadata):
    self._fs_layer_digests.append({
      _BLOB_SUM_KEY: layer_digest,
    })
    self._history.append({
      _V1_COMPAT_KEY: v1_json_metadata,
    })

  def build(self, json_web_key):
    """ Build the payload and sign it, returning a SignedManifest object.
    """
    payload = OrderedDict(self._base_payload)
    payload.update({
      _HISTORY_KEY: self._history,
      _FS_LAYERS_KEY: self._fs_layer_digests,
    })

    payload_str = json.dumps(payload, indent=3)

    split_point = payload_str.rfind('\n}')

    protected_payload = {
      'formatTail': jwt.utils.base64url_encode(payload_str[split_point:]),
      'formatLength': split_point,
      'time': datetime.utcnow().strftime(ISO_DATETIME_FORMAT_ZULU),
    }
    protected = jwt.utils.base64url_encode(json.dumps(protected_payload))
    logger.debug('Generated protected block: %s', protected)

    bytes_to_sign = '{0}.{1}'.format(protected, jwt.utils.base64url_encode(payload_str))

    signer = SIGNER_ALGS[JWS_ALGORITHM]
    signature = jwt.utils.base64url_encode(signer.sign(bytes_to_sign, json_web_key.get_key()))
    logger.debug('Generated signature: %s', signature)

    signature_block = {
      'header': {
        'jwk': json_web_key.to_dict(),
        'alg': JWS_ALGORITHM,
      },
      'signature': signature,
      _PROTECTED_KEY: protected,
    }

    logger.debug('Encoded signature block: %s', json.dumps(signature_block))

    payload.update({
      _SIGNATURES_KEY: [signature_block],
    })

    return SignedManifest(json.dumps(payload, indent=3))


@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
@process_jwt_auth
@require_repo_read
@anon_protect
def fetch_manifest_by_tagname(namespace, repo_name, manifest_ref):
  try:
    manifest = model.tag.load_tag_manifest(namespace, repo_name, manifest_ref)
  except model.InvalidManifestException:
    try:
      manifest = _generate_and_store_manifest(namespace, repo_name, manifest_ref)
    except model.DataModelException:
      logger.exception('Exception when generating manifest for %s/%s:%s', namespace, repo_name,
                       manifest_ref)
      raise ManifestUnknown()

  return make_response(manifest.json_data, 200)


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
@process_jwt_auth
@require_repo_read
@anon_protect
def fetch_manifest_by_digest(namespace, repo_name, manifest_ref):
  try:
    manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
  except model.InvalidManifestException:
    # Without a tag name to reference, we can't make an attempt to generate the manifest
    raise ManifestUnknown()

  return make_response(manifest.json_data, 200)


@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
@process_jwt_auth
@require_repo_write
@anon_protect
def write_manifest_by_tagname(namespace, repo_name, manifest_ref):
  manifest = SignedManifest(request.data)
  if manifest.tag != manifest_ref:
    raise TagInvalid()

  return _write_manifest(namespace, repo_name, manifest)


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
@process_jwt_auth
@require_repo_write
@anon_protect
def write_manifest_by_digest(namespace, repo_name, manifest_ref):
  manifest = SignedManifest(request.data)
  if manifest.digest != manifest_ref:
    raise ManifestInvalid()

  return _write_manifest(namespace, repo_name, manifest)


def _write_manifest(namespace, repo_name, manifest):
  if manifest.namespace != namespace or manifest.repo_name != repo_name:
    raise NameInvalid()

  manifest_digest = manifest.digest
  tag_name = manifest.tag

  leaf_layer = None
  try:
    for mdata in manifest.layers:
      # Store the v1 metadata in the db
      v1_mdata = mdata.v1_metadata
      digest_str = str(mdata.digest)
      model.image.synthesize_v1_image(namespace, repo_name, digest_str, v1_mdata.docker_id,
                                      v1_mdata.created, v1_mdata.comment, v1_mdata.command,
                                      mdata.v1_metadata_str, v1_mdata.parent)
      leaf_layer = mdata

  except model.InvalidImageException:
    raise BlobUnknown(detail={'digest': digest_str})

  if leaf_layer is None:
    # The manifest doesn't actually reference any layers!
    raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})

  model.tag.store_tag_manifest(namespace, repo_name, tag_name, leaf_layer.v1_metadata.docker_id,
                               manifest_digest, request.data)

  # Spawn the repo_push event.
  repo = model.repository.get_repository(namespace, repo_name)
  if repo is not None:
    event_data = {
      'updated_tags': [tag_name],
    }

    track_and_log('push_repo', repo)
    spawn_notification(repo, 'repo_push', event_data)

  response = make_response('OK', 202)
  response.headers['Docker-Content-Digest'] = manifest_digest
  response.headers['Location'] = url_for('v2.fetch_manifest_by_digest', namespace=namespace,
                                         repo_name=repo_name, manifest_ref=manifest_digest)
  return response


@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
@process_jwt_auth
@require_repo_write
@anon_protect
def delete_manifest_by_digest(namespace, repo_name, manifest_ref):
  """ Delete the manifest specified by the digest. Note: there is no equivalent
      method for deleting by tag name because it is forbidden by the spec.
  """
  try:
    manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
  except model.InvalidManifestException:
    # Without a tag name to reference, we can't make an attempt to generate the manifest
    raise ManifestUnknown()

  manifest.delete_instance()

  return make_response('', 202)


def _generate_and_store_manifest(namespace, repo_name, tag_name):
  # First look up the tag object and its ancestors
  image = model.tag.get_tag_image(namespace, repo_name, tag_name)
  parents = model.image.get_parent_images(namespace, repo_name, image)

  # Create and populate the manifest builder
  builder = SignedManifestBuilder(namespace, repo_name, tag_name)

  # Add the leaf layer
  builder.add_layer(image.storage.checksum, __get_and_backfill_image_metadata(image))

  for parent in parents:
    builder.add_layer(parent.storage.checksum, __get_and_backfill_image_metadata(parent))

  # Sign the manifest with our signing key.
  manifest = builder.build(docker_v2_signing_key)
  manifest_row = model.tag.associate_generated_tag_manifest(namespace, repo_name, tag_name,
                                                            manifest.digest, manifest.bytes)

  return manifest_row


def __get_and_backfill_image_metadata(image):
  image_metadata = image.v1_json_metadata
  if image_metadata is None:
    logger.warning('Loading metadata from storage for image id: %s', image.id)

    image.v1_json_metadata = model.image.get_image_json(image)
    logger.info('Saving backfilled metadata for image id: %s', image.id)
    image.save()

  return image_metadata