350 lines
11 KiB
Python
350 lines
11 KiB
Python
# XXX This code is not yet ready to be run in production, and should remain disabled until such
|
|
# XXX time as this notice is removed.
|
|
|
|
import logging
|
|
import re
|
|
import jwt.utils
|
|
import json
|
|
|
|
from flask import make_response, request, url_for
|
|
from collections import namedtuple, OrderedDict
|
|
from jwkest.jws import SIGNER_ALGS
|
|
from datetime import datetime
|
|
|
|
from app import storage, docker_v2_signing_key
|
|
from auth.jwt_auth import process_jwt_auth
|
|
from endpoints.decorators import anon_protect
|
|
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
|
|
from endpoints.v2.errors import (ManifestBlobUnknown, ManifestInvalid, ManifestUnverified,
|
|
ManifestUnknown, TagInvalid, NameInvalid)
|
|
from endpoints.trackhelper import track_and_log
|
|
from endpoints.notificationhelper import spawn_notification
|
|
from digest import digest_tools
|
|
from data import model
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
VALID_TAG_PATTERN = r'[\w][\w.-]{0,127}'
|
|
|
|
BASE_MANIFEST_ROUTE = '/<namespace>/<repo_name>/manifests/<regex("{0}"):manifest_ref>'
|
|
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
|
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)
|
|
|
|
|
|
ISO_DATETIME_FORMAT_ZULU = '%Y-%m-%dT%H:%M:%SZ'
|
|
JWS_ALGORITHM = 'RS256'
|
|
|
|
|
|
ImageMetadata = namedtuple('ImageMetadata', ['digest', 'v1_metadata', 'v1_metadata_str'])
|
|
ExtractedV1Metadata = namedtuple('ExtractedV1Metadata', ['docker_id', 'parent', 'created',
|
|
'comment', 'command'])
|
|
|
|
|
|
_SIGNATURES_KEY = 'signatures'
|
|
_PROTECTED_KEY = 'protected'
|
|
_FORMAT_LENGTH_KEY = 'formatLength'
|
|
_FORMAT_TAIL_KEY = 'formatTail'
|
|
_REPO_NAME_KEY = 'name'
|
|
_REPO_TAG_KEY = 'tag'
|
|
_FS_LAYERS_KEY = 'fsLayers'
|
|
_HISTORY_KEY = 'history'
|
|
_BLOB_SUM_KEY = 'blobSum'
|
|
_V1_COMPAT_KEY = 'v1Compatibility'
|
|
_ARCH_KEY = 'architecture'
|
|
_SCHEMA_VER = 'schemaVersion'
|
|
|
|
|
|
class SignedManifest(object):
|
|
|
|
def __init__(self, manifest_bytes):
|
|
self._bytes = manifest_bytes
|
|
|
|
self._parsed = json.loads(manifest_bytes)
|
|
self._signatures = self._parsed[_SIGNATURES_KEY]
|
|
self._namespace, self._repo_name = self._parsed[_REPO_NAME_KEY].split('/')
|
|
self._tag = self._parsed[_REPO_TAG_KEY]
|
|
|
|
self._validate()
|
|
|
|
def _validate(self):
|
|
pass
|
|
|
|
@property
|
|
def signatures(self):
|
|
return self._signatures
|
|
|
|
@property
|
|
def namespace(self):
|
|
return self._namespace
|
|
|
|
@property
|
|
def repo_name(self):
|
|
return self._repo_name
|
|
|
|
@property
|
|
def tag(self):
|
|
return self._tag
|
|
|
|
@property
|
|
def bytes(self):
|
|
return self._bytes
|
|
|
|
@property
|
|
def digest(self):
|
|
return digest_tools.sha256_digest(self.payload)
|
|
|
|
@property
|
|
def layers(self):
|
|
""" Returns a generator of objects that have the blobSum and v1Compatibility keys in them,
|
|
starting from the root image and working toward the leaf node.
|
|
"""
|
|
for blob_sum_obj, history_obj in reversed(zip(self._parsed[_FS_LAYERS_KEY],
|
|
self._parsed[_HISTORY_KEY])):
|
|
image_digest = digest_tools.Digest.parse_digest(blob_sum_obj[_BLOB_SUM_KEY])
|
|
metadata_string = history_obj[_V1_COMPAT_KEY]
|
|
|
|
v1_metadata = json.loads(metadata_string)
|
|
command_list = v1_metadata.get('container_config', {}).get('Cmd', None)
|
|
command = json.dumps(command_list) if command_list else None
|
|
|
|
extracted = ExtractedV1Metadata(v1_metadata['id'], v1_metadata.get('parent'),
|
|
v1_metadata.get('created'), v1_metadata.get('comment'),
|
|
command)
|
|
yield ImageMetadata(image_digest, extracted, metadata_string)
|
|
|
|
@property
|
|
def payload(self):
|
|
protected = str(self._signatures[0][_PROTECTED_KEY])
|
|
|
|
parsed_protected = json.loads(jwt.utils.base64url_decode(protected))
|
|
logger.debug('parsed_protected: %s', parsed_protected)
|
|
|
|
signed_content_head = self._bytes[:parsed_protected[_FORMAT_LENGTH_KEY]]
|
|
logger.debug('signed content head: %s', signed_content_head)
|
|
|
|
signed_content_tail = jwt.utils.base64url_decode(str(parsed_protected[_FORMAT_TAIL_KEY]))
|
|
logger.debug('signed content tail: %s', signed_content_tail)
|
|
return signed_content_head + signed_content_tail
|
|
|
|
|
|
class SignedManifestBuilder(object):
|
|
""" Class which represents a manifest which is currently being built.
|
|
"""
|
|
def __init__(self, namespace, repo_name, tag, architecture='amd64', schema_ver=1):
|
|
self._base_payload = {
|
|
_REPO_TAG_KEY: tag,
|
|
_REPO_NAME_KEY: '{0}/{1}'.format(namespace, repo_name),
|
|
_ARCH_KEY: architecture,
|
|
_SCHEMA_VER: schema_ver,
|
|
}
|
|
|
|
self._fs_layer_digests = []
|
|
self._history = []
|
|
|
|
def add_layer(self, layer_digest, v1_json_metadata):
|
|
self._fs_layer_digests.append({
|
|
_BLOB_SUM_KEY: layer_digest,
|
|
})
|
|
self._history.append({
|
|
_V1_COMPAT_KEY: v1_json_metadata,
|
|
})
|
|
|
|
def build(self, json_web_key):
|
|
""" Build the payload and sign it, returning a SignedManifest object.
|
|
"""
|
|
payload = OrderedDict(self._base_payload)
|
|
payload.update({
|
|
_HISTORY_KEY: self._history,
|
|
_FS_LAYERS_KEY: self._fs_layer_digests,
|
|
})
|
|
|
|
payload_str = json.dumps(payload, indent=3)
|
|
|
|
split_point = payload_str.rfind('\n}')
|
|
|
|
protected_payload = {
|
|
'formatTail': jwt.utils.base64url_encode(payload_str[split_point:]),
|
|
'formatLength': split_point,
|
|
'time': datetime.utcnow().strftime(ISO_DATETIME_FORMAT_ZULU),
|
|
}
|
|
protected = jwt.utils.base64url_encode(json.dumps(protected_payload))
|
|
logger.debug('Generated protected block: %s', protected)
|
|
|
|
bytes_to_sign = '{0}.{1}'.format(protected, jwt.utils.base64url_encode(payload_str))
|
|
|
|
signer = SIGNER_ALGS[JWS_ALGORITHM]
|
|
signature = jwt.utils.base64url_encode(signer.sign(bytes_to_sign, json_web_key.get_key()))
|
|
logger.debug('Generated signature: %s', signature)
|
|
|
|
signature_block = {
|
|
'header': {
|
|
'jwk': json_web_key.to_dict(),
|
|
'alg': JWS_ALGORITHM,
|
|
},
|
|
'signature': signature,
|
|
_PROTECTED_KEY: protected,
|
|
}
|
|
|
|
logger.debug('Encoded signature block: %s', json.dumps(signature_block))
|
|
|
|
payload.update({
|
|
_SIGNATURES_KEY: [signature_block],
|
|
})
|
|
|
|
return SignedManifest(json.dumps(payload, indent=3))
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
|
|
@process_jwt_auth
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_tagname(namespace, repo_name, manifest_ref):
|
|
try:
|
|
manifest = model.tag.load_tag_manifest(namespace, repo_name, manifest_ref)
|
|
except model.InvalidManifestException:
|
|
try:
|
|
manifest = _generate_and_store_manifest(namespace, repo_name, manifest_ref)
|
|
except model.DataModelException:
|
|
logger.exception('Exception when generating manifest for %s/%s:%s', namespace, repo_name,
|
|
manifest_ref)
|
|
raise ManifestUnknown()
|
|
|
|
return make_response(manifest.json_data, 200)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
|
|
@process_jwt_auth
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_digest(namespace, repo_name, manifest_ref):
|
|
try:
|
|
manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
|
|
except model.InvalidManifestException:
|
|
# Without a tag name to reference, we can't make an attempt to generate the manifest
|
|
raise ManifestUnknown()
|
|
|
|
return make_response(manifest.json_data, 200)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
|
|
@process_jwt_auth
|
|
@require_repo_write
|
|
@anon_protect
|
|
def write_manifest_by_tagname(namespace, repo_name, manifest_ref):
|
|
manifest = SignedManifest(request.data)
|
|
if manifest.tag != manifest_ref:
|
|
raise TagInvalid()
|
|
|
|
return _write_manifest(namespace, repo_name, manifest)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
|
|
@process_jwt_auth
|
|
@require_repo_write
|
|
@anon_protect
|
|
def write_manifest_by_digest(namespace, repo_name, manifest_ref):
|
|
manifest = SignedManifest(request.data)
|
|
if manifest.digest != manifest_ref:
|
|
raise ManifestInvalid()
|
|
|
|
return _write_manifest(namespace, repo_name, manifest)
|
|
|
|
|
|
def _write_manifest(namespace, repo_name, manifest):
|
|
if manifest.namespace != namespace or manifest.repo_name != repo_name:
|
|
raise NameInvalid()
|
|
|
|
manifest_digest = manifest.digest
|
|
tag_name = manifest.tag
|
|
|
|
leaf_layer = None
|
|
try:
|
|
for mdata in manifest.layers:
|
|
# Store the v1 metadata in the db
|
|
v1_mdata = mdata.v1_metadata
|
|
digest_str = str(mdata.digest)
|
|
model.image.synthesize_v1_image(namespace, repo_name, digest_str, v1_mdata.docker_id,
|
|
v1_mdata.created, v1_mdata.comment, v1_mdata.command,
|
|
mdata.v1_metadata_str, v1_mdata.parent)
|
|
leaf_layer = mdata
|
|
|
|
except model.InvalidImageException:
|
|
raise ManifestBlobUnknown(detail={'missing': digest_str})
|
|
|
|
if leaf_layer is None:
|
|
# The manifest doesn't actually reference any layers!
|
|
raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})
|
|
|
|
model.tag.store_tag_manifest(namespace, repo_name, tag_name, leaf_layer.v1_metadata.docker_id,
|
|
manifest_digest, request.data)
|
|
|
|
# Spawn the repo_push event.
|
|
repo = model.repository.get_repository(namespace, repo_name)
|
|
if repo is not None:
|
|
event_data = {
|
|
'updated_tags': [tag_name],
|
|
}
|
|
|
|
track_and_log('push_repo', repo)
|
|
spawn_notification(repo, 'repo_push', event_data)
|
|
|
|
response = make_response('OK', 202)
|
|
response.headers['Docker-Content-Digest'] = manifest_digest
|
|
response.headers['Location'] = url_for('v2.fetch_manifest_by_digest', namespace=namespace,
|
|
repo_name=repo_name, manifest_ref=manifest_digest)
|
|
return response
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
|
|
@process_jwt_auth
|
|
@require_repo_write
|
|
@anon_protect
|
|
def delete_manifest_by_digest(namespace, repo_name, manifest_ref):
|
|
""" Delete the manifest specified by the digest. Note: there is no equivalent
|
|
method for deleting by tag name because it is forbidden by the spec.
|
|
"""
|
|
try:
|
|
manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
|
|
except model.InvalidManifestException:
|
|
# Without a tag name to reference, we can't make an attempt to generate the manifest
|
|
raise ManifestUnknown()
|
|
|
|
manifest.delete_instance()
|
|
|
|
return make_response('', 202)
|
|
|
|
|
|
def _generate_and_store_manifest(namespace, repo_name, tag_name):
|
|
# First look up the tag object and its ancestors
|
|
image = model.tag.get_tag_image(namespace, repo_name, tag_name)
|
|
parents = model.image.get_parent_images(namespace, repo_name, image)
|
|
|
|
# Create and populate the manifest builder
|
|
builder = SignedManifestBuilder(namespace, repo_name, tag_name)
|
|
|
|
# Add the leaf layer
|
|
builder.add_layer(image.storage.checksum, __get_and_backfill_image_metadata(image))
|
|
|
|
for parent in parents:
|
|
builder.add_layer(parent.storage.checksum, __get_and_backfill_image_metadata(parent))
|
|
|
|
# Sign the manifest with our signing key.
|
|
manifest = builder.build(docker_v2_signing_key)
|
|
manifest_row = model.tag.associate_generated_tag_manifest(namespace, repo_name, tag_name,
|
|
manifest.digest, manifest.bytes)
|
|
|
|
return manifest_row
|
|
|
|
|
|
def __get_and_backfill_image_metadata(image):
|
|
image_metadata = image.v1_json_metadata
|
|
if image_metadata is None:
|
|
logger.warning('Loading metadata from storage for image id: %s', image.id)
|
|
|
|
image.v1_json_metadata = model.image.get_image_json(image)
|
|
logger.info('Saving backfilled metadata for image id: %s', image.id)
|
|
image.save()
|
|
|
|
return image_metadata
|