# XXX This code is not yet ready to be run in production, and should remain disabled until such # XXX time as this notice is removed. import logging import re import jwt.utils import json from flask import make_response, request, url_for from collections import namedtuple, OrderedDict from jwkest.jws import SIGNER_ALGS from jwkest.jwk import RSAKey from Crypto.PublicKey import RSA from datetime import datetime from app import storage from auth.jwt_auth import process_jwt_auth from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp, require_repo_read, require_repo_write from endpoints.v2.errors import (ManifestBlobUnknown, ManifestInvalid, ManifestUnverified, ManifestUnknown, TagInvalid, NameInvalid) from endpoints.trackhelper import track_and_log from endpoints.notificationhelper import spawn_notification from digest import digest_tools from data import model logger = logging.getLogger(__name__) VALID_TAG_PATTERN = r'[\w][\w.-]{0,127}' BASE_MANIFEST_ROUTE = '///manifests/' MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN) MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN) ISO_DATETIME_FORMAT_ZULU = '%Y-%m-%dT%H:%M:%SZ' JWS_ALGORITHM = 'RS256' ImageMetadata = namedtuple('ImageMetadata', ['digest', 'v1_metadata', 'v1_metadata_str']) ExtractedV1Metadata = namedtuple('ExtractedV1Metadata', ['docker_id', 'parent', 'created', 'comment', 'command']) _SIGNATURES_KEY = 'signatures' _PROTECTED_KEY = 'protected' _FORMAT_LENGTH_KEY = 'formatLength' _FORMAT_TAIL_KEY = 'formatTail' _REPO_NAME_KEY = 'name' _REPO_TAG_KEY = 'tag' _FS_LAYERS_KEY = 'fsLayers' _HISTORY_KEY = 'history' _BLOB_SUM_KEY = 'blobSum' _V1_COMPAT_KEY = 'v1Compatibility' _ARCH_KEY = 'architecture' _SCHEMA_VER = 'schemaVersion' class SignedManifest(object): def __init__(self, manifest_bytes): self._bytes = manifest_bytes self._parsed = json.loads(manifest_bytes) self._signatures = self._parsed[_SIGNATURES_KEY] self._namespace, self._repo_name = self._parsed[_REPO_NAME_KEY].split('/') self._tag = self._parsed[_REPO_TAG_KEY] self._validate() def _validate(self): pass @property def signatures(self): return self._signatures @property def namespace(self): return self._namespace @property def repo_name(self): return self._repo_name @property def tag(self): return self._tag @property def bytes(self): return self._bytes @property def digest(self): return digest_tools.sha256_digest(self.payload) @property def layers(self): """ Returns a generator of objects that have the blobSum and v1Compatibility keys in them, starting from the root image and working toward the leaf node. """ for blob_sum_obj, history_obj in reversed(zip(self._parsed[_FS_LAYERS_KEY], self._parsed[_HISTORY_KEY])): image_digest = digest_tools.Digest.parse_digest(blob_sum_obj[_BLOB_SUM_KEY]) metadata_string = history_obj[_V1_COMPAT_KEY] v1_metadata = json.loads(metadata_string) command_list = v1_metadata.get('container_config', {}).get('Cmd', None) command = json.dumps(command_list) if command_list else None extracted = ExtractedV1Metadata(v1_metadata['id'], v1_metadata.get('parent'), v1_metadata.get('created'), v1_metadata.get('comment'), command) yield ImageMetadata(image_digest, extracted, metadata_string) @property def payload(self): protected = str(self._signatures[0][_PROTECTED_KEY]) parsed_protected = json.loads(jwt.utils.base64url_decode(protected)) logger.debug('parsed_protected: %s', parsed_protected) signed_content_head = self._bytes[:parsed_protected[_FORMAT_LENGTH_KEY]] logger.debug('signed content head: %s', signed_content_head) signed_content_tail = jwt.utils.base64url_decode(str(parsed_protected[_FORMAT_TAIL_KEY])) logger.debug('signed content tail: %s', signed_content_tail) return signed_content_head + signed_content_tail class SignedManifestBuilder(object): """ Class which represents a manifest which is currently being built. """ def __init__(self, namespace, repo_name, tag, architecture='amd64', schema_ver=1): self._base_payload = { _REPO_TAG_KEY: tag, _REPO_NAME_KEY: '{0}/{1}'.format(namespace, repo_name), _ARCH_KEY: architecture, _SCHEMA_VER: schema_ver, } self._fs_layer_digests = [] self._history = [] def add_layer(self, layer_digest, v1_json_metadata): self._fs_layer_digests.append({ _BLOB_SUM_KEY: layer_digest, }) self._history.append({ _V1_COMPAT_KEY: v1_json_metadata, }) def build(self, json_web_key): """ Build the payload and sign it, returning a SignedManifest object. """ payload = OrderedDict(self._base_payload) payload.update({ _HISTORY_KEY: self._history, _FS_LAYERS_KEY: self._fs_layer_digests, }) payload_str = json.dumps(payload, indent=3) split_point = payload_str.rfind('\n}') protected_payload = { 'formatTail': jwt.utils.base64url_encode(payload_str[split_point:]), 'formatLength': split_point, 'time': datetime.utcnow().strftime(ISO_DATETIME_FORMAT_ZULU), } protected = jwt.utils.base64url_encode(json.dumps(protected_payload)) logger.debug('Generated protected block: %s', protected) bytes_to_sign = '{0}.{1}'.format(protected, jwt.utils.base64url_encode(payload_str)) signer = SIGNER_ALGS[JWS_ALGORITHM] signature = jwt.utils.base64url_encode(signer.sign(bytes_to_sign, json_web_key.get_key())) logger.debug('Generated signature: %s', signature) signature_block = { 'header': { 'jwk': json_web_key.to_dict(), 'alg': JWS_ALGORITHM, }, 'signature': signature, _PROTECTED_KEY: protected, } logger.debug('Encoded signature block: %s', json.dumps(signature_block)) payload.update({ _SIGNATURES_KEY: [signature_block], }) return SignedManifest(json.dumps(payload, indent=3)) @v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET']) @process_jwt_auth @require_repo_read @anon_protect def fetch_manifest_by_tagname(namespace, repo_name, manifest_ref): try: manifest = model.tag.load_tag_manifest(namespace, repo_name, manifest_ref) except model.InvalidManifestException: try: manifest = _generate_and_store_manifest(namespace, repo_name, manifest_ref) except model.DataModelException: logger.exception('Exception when generating manifest for %s/%s:%s', namespace, repo_name, manifest_ref) raise ManifestUnknown() return make_response(manifest.json_data, 200) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET']) @process_jwt_auth @require_repo_read @anon_protect def fetch_manifest_by_digest(namespace, repo_name, manifest_ref): try: manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref) except model.InvalidManifestException: # Without a tag name to reference, we can't make an attempt to generate the manifest raise ManifestUnknown() return make_response(manifest.json_data, 200) @v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT']) @process_jwt_auth @require_repo_write @anon_protect def write_manifest_by_tagname(namespace, repo_name, manifest_ref): manifest = SignedManifest(request.data) if manifest.tag != manifest_ref: raise TagInvalid() return _write_manifest(namespace, repo_name, manifest) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT']) @process_jwt_auth @require_repo_write @anon_protect def write_manifest_by_digest(namespace, repo_name, manifest_ref): manifest = SignedManifest(request.data) if manifest.digest != manifest_ref: raise ManifestInvalid() return _write_manifest(namespace, repo_name, manifest) def _write_manifest(namespace, repo_name, manifest): if manifest.namespace != namespace or manifest.repo_name != repo_name: raise NameInvalid() manifest_digest = manifest.digest tag_name = manifest.tag leaf_layer = None try: for mdata in manifest.layers: # Store the v1 metadata in the db v1_mdata = mdata.v1_metadata digest_str = str(mdata.digest) model.image.synthesize_v1_image(namespace, repo_name, digest_str, v1_mdata.docker_id, v1_mdata.created, v1_mdata.comment, v1_mdata.command, mdata.v1_metadata_str, v1_mdata.parent) leaf_layer = mdata except model.InvalidImageException: raise ManifestBlobUnknown(detail={'missing': digest_str}) if leaf_layer is None: # The manifest doesn't actually reference any layers! raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'}) model.tag.store_tag_manifest(namespace, repo_name, tag_name, leaf_layer.v1_metadata.docker_id, manifest_digest, request.data) # Spawn the repo_push event. repo = model.repository.get_repository(namespace, repo_name) if repo is not None: event_data = { 'updated_tags': [tag_name], } track_and_log('push_repo', repo) spawn_notification(repo, 'repo_push', event_data) response = make_response('OK', 202) response.headers['Docker-Content-Digest'] = manifest_digest response.headers['Location'] = url_for('v2.fetch_manifest_by_digest', namespace=namespace, repo_name=repo_name, manifest_ref=manifest_digest) return response @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE']) @process_jwt_auth @require_repo_write @anon_protect def delete_manifest_by_digest(namespace, repo_name, manifest_ref): """ Delete the manifest specified by the digest. Note: there is no equivalent method for deleting by tag name because it is forbidden by the spec. """ try: manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref) except model.InvalidManifestException: # Without a tag name to reference, we can't make an attempt to generate the manifest raise ManifestUnknown() manifest.delete_instance() return make_response('', 202) def _generate_and_store_manifest(namespace, repo_name, tag_name): # First look up the tag object and its ancestors image = model.tag.get_tag_image(namespace, repo_name, tag_name) parents = model.image.get_parent_images(namespace, repo_name, image) # Create and populate the manifest builder builder = SignedManifestBuilder(namespace, repo_name, tag_name) # Add the leaf layer builder.add_layer(image.storage.checksum, __get_and_backfill_image_metadata(image)) for parent in parents: builder.add_layer(parent.storage.checksum, __get_and_backfill_image_metadata(parent)) # TODO, stop generating a new key every time we sign a manifest, publish our key new_key = RSA.generate(2048) jwk = RSAKey(key=new_key) manifest = builder.build(jwk) manifest_row = model.tag.associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest.digest, manifest.bytes) return manifest_row def __get_and_backfill_image_metadata(image): image_metadata = image.v1_json_metadata if image_metadata is None: logger.warning('Loading metadata from storage for image id: %s', image.id) image.v1_json_metadata = model.image.get_image_json(image) logger.info('Saving backfilled metadata for image id: %s', image.id) image.save() return image_metadata