Merge pull request #885 from jakedt/python-registry-v2
Python registry v2 mega merge
This commit is contained in:
Jake Moshenko
commit
7205bf5e7f
54 changed files with 3256 additions and 837 deletions
|
|
@ -154,10 +154,7 @@ class RepositoryTagImages(RepositoryParamResource):
|
|||
image_map[str(image.id)] = image
|
||||
|
||||
image_map_all = dict(image_map)
|
||||
|
||||
parents = list(parent_images)
|
||||
parents.reverse()
|
||||
all_images = [tag_image] + parents
|
||||
all_images = [tag_image] + list(parent_images)
|
||||
|
||||
# Filter the images returned to those not found in the ancestry of any of the other tags in
|
||||
# the repository.
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from auth.auth_context import get_authenticated_user, get_grant_user_context
|
|||
from digest import checksums
|
||||
from util.registry import changes
|
||||
from util.http import abort, exact_abort
|
||||
from util.registry.filelike import SocketReader
|
||||
from auth.permissions import (ReadRepositoryPermission,
|
||||
ModifyRepositoryPermission)
|
||||
from data import model, database
|
||||
|
|
@ -24,26 +25,6 @@ from endpoints.decorators import anon_protect
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SocketReader(object):
|
||||
def __init__(self, fp):
|
||||
self._fp = fp
|
||||
self.handlers = []
|
||||
|
||||
def add_handler(self, handler):
|
||||
self.handlers.append(handler)
|
||||
|
||||
def read(self, n=-1):
|
||||
buf = self._fp.read(n)
|
||||
if not buf:
|
||||
return ''
|
||||
for handler in self.handlers:
|
||||
handler(buf)
|
||||
return buf
|
||||
|
||||
def tell(self):
|
||||
raise IOError('Stream is not seekable.')
|
||||
|
||||
|
||||
def image_is_uploading(repo_image):
|
||||
if repo_image is None:
|
||||
return False
|
||||
|
|
@ -159,11 +140,9 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
logger.debug('Looking up the layer path')
|
||||
try:
|
||||
path = store.image_layer_path(repo_image.storage.uuid)
|
||||
|
||||
logger.debug('Looking up the direct download URL')
|
||||
path = model.storage.get_layer_path(repo_image.storage)
|
||||
logger.debug('Looking up the direct download URL for path: %s', path)
|
||||
direct_download_url = store.get_direct_download_url(repo_image.storage.locations, path)
|
||||
|
||||
if direct_download_url:
|
||||
|
|
@ -205,8 +184,9 @@ def put_image_layer(namespace, repository, image_id):
|
|||
logger.exception('Exception when retrieving image data')
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
||||
|
||||
logger.debug('Retrieving image path info')
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
uuid = repo_image.storage.uuid
|
||||
layer_path = store.v1_image_layer_path(uuid)
|
||||
logger.info('Storing layer at v1 path: %s', layer_path)
|
||||
|
||||
if (store.exists(repo_image.storage.locations, layer_path) and not
|
||||
image_is_uploading(repo_image)):
|
||||
|
|
@ -393,37 +373,22 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
if not permission.can() and not model.repository.repository_is_public(namespace, repository):
|
||||
abort(403)
|
||||
|
||||
logger.debug('Looking up repo image')
|
||||
repo_image = model.image.get_repo_image_extended(namespace, repository, image_id)
|
||||
|
||||
logger.debug('Looking up image data')
|
||||
try:
|
||||
uuid = repo_image.storage.uuid
|
||||
data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
|
||||
except (IOError, AttributeError):
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
image = model.image.get_image_by_id(namespace, repository, image_id)
|
||||
except model.InvalidImageException:
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
||||
|
||||
logger.debug('Converting to <-> from JSON')
|
||||
response = make_response(json.dumps(json.loads(data)), 200)
|
||||
parents = model.image.get_parent_images(namespace, repository, image)
|
||||
|
||||
ancestry_docker_ids = [image.docker_image_id]
|
||||
ancestry_docker_ids.extend([parent.docker_image_id for parent in parents])
|
||||
|
||||
# We can not use jsonify here because we are returning a list not an object
|
||||
response = make_response(json.dumps(ancestry_docker_ids), 200)
|
||||
response.headers.extend(headers)
|
||||
|
||||
logger.debug('Done')
|
||||
return response
|
||||
|
||||
|
||||
def generate_ancestry(image_id, uuid, locations, parent_id=None, parent_uuid=None,
|
||||
parent_locations=None):
|
||||
if not parent_id:
|
||||
store.put_content(locations, store.image_ancestry_path(uuid), json.dumps([image_id]))
|
||||
return
|
||||
|
||||
data = store.get_content(parent_locations, store.image_ancestry_path(parent_uuid))
|
||||
data = json.loads(data)
|
||||
data.insert(0, image_id)
|
||||
store.put_content(locations, store.image_ancestry_path(uuid), json.dumps(data))
|
||||
|
||||
|
||||
def store_checksum(image_with_storage, checksum, content_checksum):
|
||||
checksum_parts = checksum.split(':')
|
||||
if len(checksum_parts) != 2:
|
||||
|
|
@ -486,22 +451,16 @@ def put_image_json(namespace, repository, image_id):
|
|||
abort(400, 'JSON data contains invalid id for image: %(image_id)s',
|
||||
issue='invalid-request', image_id=image_id)
|
||||
|
||||
parent_id = data.get('parent')
|
||||
parent_id = data.get('parent', None)
|
||||
|
||||
parent_image = None
|
||||
if parent_id:
|
||||
logger.debug('Looking up parent image')
|
||||
parent_image = model.image.get_repo_image_extended(namespace, repository, parent_id)
|
||||
|
||||
parent_uuid = parent_image and parent_image.storage.uuid
|
||||
parent_locations = parent_image and parent_image.storage.locations
|
||||
|
||||
if parent_id:
|
||||
logger.debug('Looking up parent image data')
|
||||
|
||||
if parent_id and not parent_image.v1_json_metadata:
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
if not parent_image or parent_image.storage.uploading:
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
|
||||
logger.debug('Checking if image already exists')
|
||||
if repo_image.v1_json_metadata and not image_is_uploading(repo_image):
|
||||
|
|
@ -510,8 +469,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
set_uploading_flag(repo_image, True)
|
||||
|
||||
# If we reach that point, it means that this is a new image or a retry
|
||||
# on a failed push
|
||||
# save the metadata
|
||||
# on a failed push, save the metadata
|
||||
command_list = data.get('container_config', {}).get('Cmd', None)
|
||||
command = json.dumps(command_list) if command_list else None
|
||||
|
||||
|
|
@ -519,16 +477,6 @@ def put_image_json(namespace, repository, image_id):
|
|||
model.image.set_image_metadata(image_id, namespace, repository, data.get('created'),
|
||||
data.get('comment'), command, v1_metadata, parent_image)
|
||||
|
||||
logger.debug('Generating image ancestry')
|
||||
|
||||
try:
|
||||
generate_ancestry(image_id, repo_image.storage.uuid, repo_image.storage.locations, parent_id,
|
||||
parent_uuid, parent_locations)
|
||||
except IOError as ioe:
|
||||
logger.debug('Error when generating ancestry: %s', ioe.message)
|
||||
abort(404)
|
||||
|
||||
logger.debug('Done')
|
||||
return make_response('true', 200)
|
||||
|
||||
|
||||
|
|
@ -556,7 +504,7 @@ def process_image_changes(namespace, repository, image_id):
|
|||
parent_trie_path = None
|
||||
if parents:
|
||||
parent_trie_path, parent_locations = process_image_changes(namespace, repository,
|
||||
parents[-1].docker_image_id)
|
||||
parents[0].docker_image_id)
|
||||
|
||||
# Read in the collapsed layer state of the filesystem for the parent
|
||||
parent_trie = changes.empty_fs()
|
||||
|
|
@ -565,7 +513,7 @@ def process_image_changes(namespace, repository, image_id):
|
|||
parent_trie.frombytes(parent_trie_bytes)
|
||||
|
||||
# Read in the file entries from the layer tar file
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
layer_path = model.storage.get_layer_path(repo_image.storage)
|
||||
with store.stream_read_file(image.storage.locations, layer_path) as layer_tar_stream:
|
||||
removed_files = set()
|
||||
layer_files = changes.files_and_dirs_from_tar(layer_tar_stream,
|
||||
|
|
|
|||
|
|
@ -40,7 +40,11 @@ def get_tag(namespace, repository, tag):
|
|||
permission = ReadRepositoryPermission(namespace, repository)
|
||||
|
||||
if permission.can() or model.repository.repository_is_public(namespace, repository):
|
||||
tag_image = model.tag.get_tag_image(namespace, repository, tag)
|
||||
try:
|
||||
tag_image = model.tag.get_tag_image(namespace, repository, tag)
|
||||
except model.DataModelException:
|
||||
abort(404)
|
||||
|
||||
resp = make_response('"%s"' % tag_image.docker_image_id)
|
||||
resp.headers['Content-Type'] = 'application/json'
|
||||
return resp
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
# XXX This code is not yet ready to be run in production, and should remain disabled until such
|
||||
# XXX time as this notice is removed.
|
||||
|
||||
import logging
|
||||
|
||||
from flask import Blueprint, make_response, url_for, request
|
||||
from flask import Blueprint, make_response, url_for, request, jsonify
|
||||
from functools import wraps
|
||||
from urlparse import urlparse
|
||||
|
||||
import features
|
||||
|
||||
from app import metric_queue
|
||||
from endpoints.decorators import anon_protect, anon_allowed
|
||||
from endpoints.v2.errors import V2RegistryException
|
||||
from auth.jwt_auth import process_jwt_auth
|
||||
from auth.auth_context import get_grant_user_context
|
||||
from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
|
||||
|
|
@ -16,6 +16,8 @@ from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermissi
|
|||
from data import model
|
||||
from util.http import abort
|
||||
from util.saas.metricqueue import time_blueprint
|
||||
from util import get_app_url
|
||||
from app import app
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -23,6 +25,16 @@ v2_bp = Blueprint('v2', __name__)
|
|||
|
||||
time_blueprint(v2_bp, metric_queue)
|
||||
|
||||
@v2_bp.app_errorhandler(V2RegistryException)
|
||||
def handle_registry_v2_exception(error):
|
||||
response = jsonify({
|
||||
'errors': [error.as_dict()]
|
||||
})
|
||||
response.status_code = error.http_status_code
|
||||
logger.debug('sending response: %s', response.get_data())
|
||||
return response
|
||||
|
||||
|
||||
def _require_repo_permission(permission_class, allow_public=False):
|
||||
def wrapper(func):
|
||||
@wraps(func)
|
||||
|
|
@ -49,7 +61,20 @@ def get_input_stream(flask_request):
|
|||
return flask_request.stream
|
||||
|
||||
|
||||
# TODO remove when v2 is deployed everywhere
|
||||
def route_show_if(value):
|
||||
def decorator(f):
|
||||
@wraps(f)
|
||||
def decorated_function(*args, **kwargs):
|
||||
if not value:
|
||||
abort(404)
|
||||
|
||||
return f(*args, **kwargs)
|
||||
return decorated_function
|
||||
return decorator
|
||||
|
||||
@v2_bp.route('/')
|
||||
@route_show_if(features.ADVERTISE_V2)
|
||||
@process_jwt_auth
|
||||
@anon_allowed
|
||||
def v2_support_enabled():
|
||||
|
|
@ -57,9 +82,11 @@ def v2_support_enabled():
|
|||
|
||||
if get_grant_user_context() is None:
|
||||
response = make_response('true', 401)
|
||||
realm_hostname = urlparse(request.url).netloc
|
||||
realm_auth_path = url_for('v2.generate_registry_jwt')
|
||||
authenticate = 'Bearer realm="{0}{1}",service="quay"'.format(realm_hostname, realm_auth_path)
|
||||
|
||||
authenticate = 'Bearer realm="{0}{1}",service="{2}"'.format(get_app_url(app.config),
|
||||
realm_auth_path,
|
||||
app.config['SERVER_HOSTNAME'])
|
||||
response.headers['WWW-Authenticate'] = authenticate
|
||||
|
||||
response.headers['Docker-Distribution-API-Version'] = 'registry/2.0'
|
||||
|
|
@ -69,3 +96,5 @@ def v2_support_enabled():
|
|||
from endpoints.v2 import v2auth
|
||||
from endpoints.v2 import manifest
|
||||
from endpoints.v2 import blob
|
||||
from endpoints.v2 import tag
|
||||
from endpoints.v2 import catalog
|
||||
|
|
|
|||
|
|
@ -1,17 +1,18 @@
|
|||
# XXX This code is not yet ready to be run in production, and should remain disabled until such
|
||||
# XXX time as this notice is removed.
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from flask import make_response, url_for, request
|
||||
from flask import make_response, url_for, request, redirect, Response, abort as flask_abort
|
||||
|
||||
from app import storage, app
|
||||
from data import model
|
||||
from data import model, database
|
||||
from digest import digest_tools
|
||||
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream
|
||||
from endpoints.v2.errors import BlobUnknown, BlobUploadInvalid, BlobUploadUnknown, Unsupported
|
||||
from auth.jwt_auth import process_jwt_auth
|
||||
from endpoints.decorators import anon_protect
|
||||
from util.http import abort
|
||||
from util.cache import cache_control
|
||||
from util.registry.filelike import wrap_with_handler, StreamSlice
|
||||
from storage.basestorage import InvalidChunkException
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -19,29 +20,78 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
BASE_BLOB_ROUTE = '/<namespace>/<repo_name>/blobs/<regex("{0}"):digest>'
|
||||
BLOB_DIGEST_ROUTE = BASE_BLOB_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
||||
RANGE_HEADER_REGEX = re.compile(r'^bytes=([0-9]+)-([0-9]+)$')
|
||||
|
||||
|
||||
class _InvalidRangeHeader(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _base_blob_fetch(namespace, repo_name, digest):
|
||||
""" Some work that is common to both GET and HEAD requests. Callers MUST check for proper
|
||||
authorization before calling this method.
|
||||
"""
|
||||
try:
|
||||
found = model.blob.get_repo_blob_by_digest(namespace, repo_name, digest)
|
||||
except model.BlobDoesNotExist:
|
||||
raise BlobUnknown()
|
||||
|
||||
headers = {
|
||||
'Docker-Content-Digest': digest,
|
||||
'Content-Length': found.image_size,
|
||||
}
|
||||
|
||||
# Add the Accept-Ranges header if the storage engine supports resumable
|
||||
# downloads.
|
||||
if storage.get_supports_resumable_downloads(found.locations):
|
||||
logger.debug('Storage supports resumable downloads')
|
||||
headers['Accept-Ranges'] = 'bytes'
|
||||
|
||||
return found, headers
|
||||
|
||||
|
||||
@v2_bp.route(BLOB_DIGEST_ROUTE, methods=['HEAD'])
|
||||
@process_jwt_auth
|
||||
@require_repo_read
|
||||
@anon_protect
|
||||
def check_blob_existence(namespace, repo_name, digest):
|
||||
try:
|
||||
found = model.blob.get_repo_blob_by_digest(namespace, repo_name, digest)
|
||||
@cache_control(max_age=31436000)
|
||||
def check_blob_exists(namespace, repo_name, digest):
|
||||
_, headers = _base_blob_fetch(namespace, repo_name, digest)
|
||||
|
||||
# The response body must be empty for a successful HEAD request
|
||||
return make_response('')
|
||||
except model.BlobDoesNotExist:
|
||||
abort(404)
|
||||
response = make_response('')
|
||||
response.headers.extend(headers)
|
||||
response.headers['Content-Length'] = headers['Content-Length']
|
||||
return response
|
||||
|
||||
|
||||
@v2_bp.route(BLOB_DIGEST_ROUTE, methods=['GET'])
|
||||
@process_jwt_auth
|
||||
@require_repo_read
|
||||
@anon_protect
|
||||
@cache_control(max_age=31536000)
|
||||
def download_blob(namespace, repo_name, digest):
|
||||
# TODO Implement this
|
||||
return make_response('')
|
||||
found, headers = _base_blob_fetch(namespace, repo_name, digest)
|
||||
|
||||
path = model.storage.get_layer_path(found)
|
||||
logger.debug('Looking up the direct download URL for path: %s', path)
|
||||
direct_download_url = storage.get_direct_download_url(found.locations, path)
|
||||
|
||||
if direct_download_url:
|
||||
logger.debug('Returning direct download URL')
|
||||
resp = redirect(direct_download_url)
|
||||
resp.headers.extend(headers)
|
||||
return resp
|
||||
|
||||
logger.debug('Streaming layer data')
|
||||
|
||||
# Close the database handle here for this process before we send the long download.
|
||||
database.close_db_filter(None)
|
||||
|
||||
return Response(storage.stream_read(found.locations, path), headers=headers)
|
||||
|
||||
|
||||
def _render_range(num_uploaded_bytes, with_bytes_prefix=True):
|
||||
return '{0}0-{1}'.format('bytes=' if with_bytes_prefix else '', num_uploaded_bytes - 1)
|
||||
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/blobs/uploads/', methods=['POST'])
|
||||
|
|
@ -49,12 +99,162 @@ def download_blob(namespace, repo_name, digest):
|
|||
@require_repo_write
|
||||
@anon_protect
|
||||
def start_blob_upload(namespace, repo_name):
|
||||
new_upload_uuid = storage.initiate_chunked_upload(storage.preferred_locations[0])
|
||||
accepted = make_response('', 202)
|
||||
accepted.headers['Location'] = url_for('v2.upload_chunk', namespace=namespace,
|
||||
repo_name=repo_name, upload_uuid=new_upload_uuid)
|
||||
accepted.headers['Range'] = 'bytes=0-0'
|
||||
accepted.headers['Docker-Upload-UUID'] = new_upload_uuid
|
||||
location_name = storage.preferred_locations[0]
|
||||
new_upload_uuid, upload_metadata = storage.initiate_chunked_upload(location_name)
|
||||
model.blob.initiate_upload(namespace, repo_name, new_upload_uuid, location_name, upload_metadata)
|
||||
|
||||
digest = request.args.get('digest', None)
|
||||
if digest is None:
|
||||
# The user will send the blob data in another request
|
||||
accepted = make_response('', 202)
|
||||
accepted.headers['Location'] = url_for('v2.upload_chunk', namespace=namespace,
|
||||
repo_name=repo_name, upload_uuid=new_upload_uuid)
|
||||
accepted.headers['Range'] = _render_range(0)
|
||||
accepted.headers['Docker-Upload-UUID'] = new_upload_uuid
|
||||
return accepted
|
||||
else:
|
||||
# The user plans to send us the entire body right now
|
||||
uploaded = _upload_chunk(namespace, repo_name, new_upload_uuid)
|
||||
uploaded.save()
|
||||
|
||||
return _finish_upload(namespace, repo_name, uploaded, digest)
|
||||
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/blobs/uploads/<upload_uuid>', methods=['GET'])
|
||||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def fetch_existing_upload(namespace, repo_name, upload_uuid):
|
||||
try:
|
||||
found = model.blob.get_blob_upload(namespace, repo_name, upload_uuid)
|
||||
except model.InvalidBlobUpload:
|
||||
raise BlobUploadUnknown()
|
||||
|
||||
# Note: Docker byte ranges are exclusive so we have to add one to the byte count.
|
||||
accepted = make_response('', 204)
|
||||
accepted.headers['Range'] = _render_range(found.byte_count + 1)
|
||||
accepted.headers['Docker-Upload-UUID'] = upload_uuid
|
||||
return accepted
|
||||
|
||||
|
||||
def _current_request_path():
|
||||
return '{0}{1}'.format(request.script_root, request.path)
|
||||
|
||||
|
||||
def _range_not_satisfiable(valid_end):
|
||||
invalid_range = make_response('', 416)
|
||||
invalid_range.headers['Location'] = _current_request_path()
|
||||
invalid_range.headers['Range'] = '0-{0}'.format(valid_end)
|
||||
invalid_range.headers['Docker-Upload-UUID'] = request.view_args['upload_uuid']
|
||||
flask_abort(invalid_range)
|
||||
|
||||
|
||||
def _parse_range_header(range_header_text):
|
||||
""" Parses the range header, and returns a tuple of the start offset and the length,
|
||||
or raises an _InvalidRangeHeader exception.
|
||||
"""
|
||||
found = RANGE_HEADER_REGEX.match(range_header_text)
|
||||
if found is None:
|
||||
raise _InvalidRangeHeader()
|
||||
|
||||
start = int(found.group(1))
|
||||
length = int(found.group(2)) - start
|
||||
|
||||
if length <= 0:
|
||||
raise _InvalidRangeHeader()
|
||||
|
||||
return (start, length)
|
||||
|
||||
|
||||
def _upload_chunk(namespace, repo_name, upload_uuid):
|
||||
""" Common code among the various uploading paths for appending data to blobs.
|
||||
Callers MUST call .save() or .delete_instance() on the returned database object.
|
||||
"""
|
||||
try:
|
||||
found = model.blob.get_blob_upload(namespace, repo_name, upload_uuid)
|
||||
except model.InvalidBlobUpload:
|
||||
raise BlobUploadUnknown()
|
||||
|
||||
start_offset, length = 0, -1
|
||||
range_header = request.headers.get('range', None)
|
||||
if range_header is not None:
|
||||
try:
|
||||
start_offset, length = _parse_range_header(range_header)
|
||||
except _InvalidRangeHeader:
|
||||
_range_not_satisfiable(found.byte_count)
|
||||
|
||||
if start_offset > 0 and start_offset > found.byte_count:
|
||||
_range_not_satisfiable(found.byte_count)
|
||||
|
||||
input_fp = get_input_stream(request)
|
||||
|
||||
if start_offset > 0 and start_offset < found.byte_count:
|
||||
# Skip the bytes which were received on a previous push, which are already stored and
|
||||
# included in the sha calculation
|
||||
input_fp = StreamSlice(input_fp, found.byte_count - start_offset)
|
||||
start_offset = found.byte_count
|
||||
|
||||
input_fp = wrap_with_handler(input_fp, found.sha_state.update)
|
||||
|
||||
try:
|
||||
length_written, new_metadata = storage.stream_upload_chunk({found.location.name}, upload_uuid,
|
||||
start_offset, length, input_fp,
|
||||
found.storage_metadata)
|
||||
except InvalidChunkException:
|
||||
_range_not_satisfiable(found.byte_count)
|
||||
|
||||
found.storage_metadata = new_metadata
|
||||
found.byte_count += length_written
|
||||
return found
|
||||
|
||||
|
||||
def _finish_upload(namespace, repo_name, upload_obj, expected_digest):
|
||||
# Verify that the digest's SHA matches that of the uploaded data.
|
||||
computed_digest = digest_tools.sha256_digest_from_hashlib(upload_obj.sha_state)
|
||||
if not digest_tools.digests_equal(computed_digest, expected_digest):
|
||||
raise BlobUploadInvalid()
|
||||
|
||||
# Move the storage into place, or if this was a re-upload, cancel it
|
||||
final_blob_location = digest_tools.content_path(expected_digest)
|
||||
|
||||
if storage.exists({upload_obj.location.name}, final_blob_location):
|
||||
# It already existed, clean up our upload which served as proof that we had the file
|
||||
storage.cancel_chunked_upload({upload_obj.location.name}, upload_obj.uuid,
|
||||
upload_obj.storage_metadata)
|
||||
|
||||
else:
|
||||
# We were the first ones to upload this image (at least to this location)
|
||||
# Let's copy it into place
|
||||
storage.complete_chunked_upload({upload_obj.location.name}, upload_obj.uuid,
|
||||
final_blob_location, upload_obj.storage_metadata)
|
||||
|
||||
# Mark the blob as uploaded.
|
||||
model.blob.store_blob_record_and_temp_link(namespace, repo_name, expected_digest,
|
||||
upload_obj.location, upload_obj.byte_count,
|
||||
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
|
||||
|
||||
# Delete the upload tracking row.
|
||||
upload_obj.delete_instance()
|
||||
|
||||
response = make_response('', 201)
|
||||
response.headers['Docker-Content-Digest'] = expected_digest
|
||||
response.headers['Location'] = url_for('v2.download_blob', namespace=namespace,
|
||||
repo_name=repo_name, digest=expected_digest)
|
||||
return response
|
||||
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/blobs/uploads/<upload_uuid>', methods=['PATCH'])
|
||||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def upload_chunk(namespace, repo_name, upload_uuid):
|
||||
upload = _upload_chunk(namespace, repo_name, upload_uuid)
|
||||
upload.save()
|
||||
|
||||
accepted = make_response('', 204)
|
||||
accepted.headers['Location'] = _current_request_path()
|
||||
accepted.headers['Range'] = _render_range(upload.byte_count, with_bytes_prefix=False)
|
||||
accepted.headers['Docker-Upload-UUID'] = upload_uuid
|
||||
return accepted
|
||||
|
||||
|
||||
|
|
@ -62,22 +262,40 @@ def start_blob_upload(namespace, repo_name):
|
|||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def upload_chunk(namespace, repo_name, upload_uuid):
|
||||
def monolithic_upload_or_last_chunk(namespace, repo_name, upload_uuid):
|
||||
digest = request.args.get('digest', None)
|
||||
upload_location = storage.preferred_locations[0]
|
||||
bytes_written = storage.stream_upload_chunk(upload_location, upload_uuid, 0, -1,
|
||||
get_input_stream(request))
|
||||
if digest is None:
|
||||
raise BlobUploadInvalid()
|
||||
|
||||
if digest is not None:
|
||||
final_blob_location = digest_tools.content_path(digest)
|
||||
storage.complete_chunked_upload(upload_location, upload_uuid, final_blob_location, digest)
|
||||
model.blob.store_blob_record_and_temp_link(namespace, repo_name, digest, upload_location,
|
||||
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
|
||||
found = _upload_chunk(namespace, repo_name, upload_uuid)
|
||||
return _finish_upload(namespace, repo_name, found, digest)
|
||||
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/blobs/uploads/<upload_uuid>', methods=['DELETE'])
|
||||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def cancel_upload(namespace, repo_name, upload_uuid):
|
||||
try:
|
||||
found = model.blob.get_blob_upload(namespace, repo_name, upload_uuid)
|
||||
except model.InvalidBlobUpload:
|
||||
raise BlobUploadUnknown()
|
||||
|
||||
# We delete the record for the upload first, since if the partial upload in
|
||||
# storage fails to delete, it doesn't break anything
|
||||
found.delete_instance()
|
||||
storage.cancel_chunked_upload({found.location.name}, found.uuid, found.storage_metadata)
|
||||
|
||||
return make_response('', 204)
|
||||
|
||||
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/blobs/<digest>', methods=['DELETE'])
|
||||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def delete_digest(namespace, repo_name, upload_uuid):
|
||||
# We do not support deleting arbitrary digests, as they break repo images.
|
||||
raise Unsupported()
|
||||
|
||||
response = make_response('', 201)
|
||||
response.headers['Docker-Content-Digest'] = digest
|
||||
response.headers['Location'] = url_for('v2.download_blob', namespace=namespace,
|
||||
repo_name=repo_name, digest=digest)
|
||||
return response
|
||||
|
||||
return make_response('', 202)
|
||||
|
|
|
|||
29
endpoints/v2/catalog.py
Normal file
29
endpoints/v2/catalog.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
from flask import jsonify, url_for
|
||||
|
||||
from endpoints.v2 import v2_bp
|
||||
from auth.auth import process_auth
|
||||
from endpoints.decorators import anon_protect
|
||||
from data import model
|
||||
from endpoints.v2.v2util import add_pagination
|
||||
from auth.auth_context import get_authenticated_user
|
||||
|
||||
@v2_bp.route('/_catalog', methods=['GET'])
|
||||
@process_auth
|
||||
@anon_protect
|
||||
def catalog_search():
|
||||
url = url_for('v2.catalog_search')
|
||||
|
||||
username = get_authenticated_user().username if get_authenticated_user() else None
|
||||
query = model.repository.get_visible_repositories(username, include_public=(username is None),
|
||||
limit=50)
|
||||
|
||||
link, query = add_pagination(query, url)
|
||||
|
||||
response = jsonify({
|
||||
'repositories': ['%s/%s' % (repo.namespace_user.username, repo.name) for repo in query],
|
||||
})
|
||||
|
||||
if link is not None:
|
||||
response.headers['Link'] = link
|
||||
|
||||
return response
|
||||
118
endpoints/v2/errors.py
Normal file
118
endpoints/v2/errors.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
class V2RegistryException(Exception):
|
||||
def __init__(self, error_code_str, message, detail, http_status_code=400):
|
||||
super(V2RegistryException, self).__init__(message)
|
||||
self.http_status_code = http_status_code
|
||||
|
||||
self._error_code_str = error_code_str
|
||||
self._detail = detail
|
||||
|
||||
def as_dict(self):
|
||||
return {
|
||||
'code': self._error_code_str,
|
||||
'message': self.message,
|
||||
'detail': self._detail if self._detail is not None else {},
|
||||
}
|
||||
|
||||
|
||||
class BlobUnknown(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(BlobUnknown, self).__init__('BLOB_UNKNOWN',
|
||||
'blob unknown to registry',
|
||||
detail,
|
||||
404)
|
||||
|
||||
|
||||
class BlobUploadInvalid(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(BlobUploadInvalid, self).__init__('BLOB_UPLOAD_INVALID',
|
||||
'blob upload invalid',
|
||||
detail)
|
||||
|
||||
|
||||
class BlobUploadUnknown(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(BlobUploadUnknown, self).__init__('BLOB_UPLOAD_UNKNOWN',
|
||||
'blob upload unknown to registry',
|
||||
detail,
|
||||
404)
|
||||
|
||||
|
||||
class DigestInvalid(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(DigestInvalid, self).__init__('DIGEST_INVALID',
|
||||
'provided digest did not match uploaded content',
|
||||
detail)
|
||||
|
||||
|
||||
class ManifestBlobUnknown(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(ManifestBlobUnknown, self).__init__('MANIFEST_BLOB_UNKNOWN',
|
||||
'manifest blob unknown to registry',
|
||||
detail)
|
||||
|
||||
|
||||
class ManifestInvalid(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(ManifestInvalid, self).__init__('MANIFEST_INVALID',
|
||||
'manifest invalid',
|
||||
detail)
|
||||
|
||||
|
||||
class ManifestUnknown(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(ManifestUnknown, self).__init__('MANIFEST_UNKNOWN',
|
||||
'manifest unknown',
|
||||
detail,
|
||||
404)
|
||||
|
||||
|
||||
class ManifestUnverified(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(ManifestUnverified, self).__init__('MANIFEST_UNVERIFIED',
|
||||
'manifest failed signature verification',
|
||||
detail)
|
||||
|
||||
|
||||
class NameInvalid(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(NameInvalid, self).__init__('NAME_INVALID',
|
||||
'invalid repository name',
|
||||
detail)
|
||||
|
||||
|
||||
class NameUnknown(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(NameUnknown, self).__init__('NAME_UNKNOWN',
|
||||
'repository name not known to registry',
|
||||
detail,
|
||||
404)
|
||||
|
||||
|
||||
class SizeInvalid(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(SizeInvalid, self).__init__('SIZE_INVALID',
|
||||
'provided length did not match content length',
|
||||
detail)
|
||||
|
||||
|
||||
class TagInvalid(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(TagInvalid, self).__init__('TAG_INVALID',
|
||||
'manifest tag did not match URI',
|
||||
detail)
|
||||
|
||||
|
||||
class Unauthorized(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(Unauthorized, self).__init__('UNAUTHORIZED',
|
||||
'access to the requested resource is not authorized',
|
||||
detail,
|
||||
401)
|
||||
|
||||
|
||||
class Unsupported(V2RegistryException):
|
||||
def __init__(self, detail=None):
|
||||
super(Unsupported, self).__init__('UNSUPPORTED',
|
||||
'The operation is unsupported.',
|
||||
detail,
|
||||
405)
|
||||
|
|
@ -1,47 +1,79 @@
|
|||
# XXX This code is not yet ready to be run in production, and should remain disabled until such
|
||||
# XXX time as this notice is removed.
|
||||
|
||||
import logging
|
||||
import re
|
||||
import jwt.utils
|
||||
import yaml
|
||||
import json
|
||||
|
||||
from flask import make_response, request
|
||||
from flask import make_response, request, url_for
|
||||
from collections import namedtuple, OrderedDict
|
||||
from jwkest.jws import SIGNER_ALGS, keyrep
|
||||
from datetime import datetime
|
||||
|
||||
from app import storage
|
||||
from app import docker_v2_signing_key
|
||||
from auth.jwt_auth import process_jwt_auth
|
||||
from endpoints.decorators import anon_protect
|
||||
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream
|
||||
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
|
||||
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnverified,
|
||||
ManifestUnknown, TagInvalid, NameInvalid)
|
||||
from endpoints.trackhelper import track_and_log
|
||||
from endpoints.notificationhelper import spawn_notification
|
||||
from digest import digest_tools
|
||||
from data import model
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
VALID_TAG_PATTERN = r'[\w][\w.-]{0,127}'
|
||||
VALID_TAG_REGEX = re.compile(VALID_TAG_PATTERN)
|
||||
|
||||
BASE_MANIFEST_ROUTE = '/<namespace>/<repo_name>/manifests/<regex("{0}"):manifest_ref>'
|
||||
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
||||
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)
|
||||
|
||||
|
||||
ISO_DATETIME_FORMAT_ZULU = '%Y-%m-%dT%H:%M:%SZ'
|
||||
JWS_ALGORITHM = 'RS256'
|
||||
|
||||
|
||||
ImageMetadata = namedtuple('ImageMetadata', ['digest', 'v1_metadata', 'v1_metadata_str'])
|
||||
ExtractedV1Metadata = namedtuple('ExtractedV1Metadata', ['docker_id', 'parent', 'created',
|
||||
'comment', 'command'])
|
||||
|
||||
|
||||
_SIGNATURES_KEY = 'signatures'
|
||||
_PROTECTED_KEY = 'protected'
|
||||
_FORMAT_LENGTH_KEY = 'formatLength'
|
||||
_FORMAT_TAIL_KEY = 'formatTail'
|
||||
_REPO_NAME_KEY = 'name'
|
||||
_REPO_TAG_KEY = 'tag'
|
||||
_FS_LAYERS_KEY = 'fsLayers'
|
||||
_HISTORY_KEY = 'history'
|
||||
_BLOB_SUM_KEY = 'blobSum'
|
||||
_V1_COMPAT_KEY = 'v1Compatibility'
|
||||
_ARCH_KEY = 'architecture'
|
||||
_SCHEMA_VER = 'schemaVersion'
|
||||
|
||||
|
||||
class SignedManifest(object):
|
||||
SIGNATURES_KEY = 'signatures'
|
||||
PROTECTED_KEY = 'protected'
|
||||
FORMAT_LENGTH_KEY = 'formatLength'
|
||||
FORMAT_TAIL_KEY = 'formatTail'
|
||||
REPO_NAME_KEY = 'name'
|
||||
REPO_TAG_KEY = 'tag'
|
||||
|
||||
def __init__(self, manifest_bytes):
|
||||
self._bytes = manifest_bytes
|
||||
parsed = yaml.safe_load(manifest_bytes)
|
||||
|
||||
self._signatures = parsed[self.SIGNATURES_KEY]
|
||||
self._namespace, self._repo_name = parsed[self.REPO_NAME_KEY].split('/')
|
||||
self._tag = parsed[self.REPO_TAG_KEY]
|
||||
self._parsed = json.loads(manifest_bytes)
|
||||
self._signatures = self._parsed[_SIGNATURES_KEY]
|
||||
self._namespace, self._repo_name = self._parsed[_REPO_NAME_KEY].split('/')
|
||||
self._tag = self._parsed[_REPO_TAG_KEY]
|
||||
|
||||
self._validate()
|
||||
|
||||
def _validate(self):
|
||||
pass
|
||||
for signature in self._signatures:
|
||||
bytes_to_verify = '{0}.{1}'.format(signature['protected'], jwt.utils.base64url_encode(self.payload))
|
||||
signer = SIGNER_ALGS[signature['header']['alg']]
|
||||
key = keyrep(signature['header']['jwk'])
|
||||
gk = key.get_key()
|
||||
sig = jwt.utils.base64url_decode(signature['signature'].encode('utf-8'))
|
||||
verified = signer.verify(bytes_to_verify, sig, gk)
|
||||
if not verified:
|
||||
raise ValueError('manifest file failed signature verification')
|
||||
|
||||
@property
|
||||
def signatures(self):
|
||||
|
|
@ -59,52 +91,306 @@ class SignedManifest(object):
|
|||
def tag(self):
|
||||
return self._tag
|
||||
|
||||
@property
|
||||
def bytes(self):
|
||||
return self._bytes
|
||||
|
||||
@property
|
||||
def digest(self):
|
||||
return digest_tools.sha256_digest(self.payload)
|
||||
|
||||
@property
|
||||
def layers(self):
|
||||
""" Returns a generator of objects that have the blobSum and v1Compatibility keys in them,
|
||||
starting from the leaf image and working toward the base node.
|
||||
"""
|
||||
for blob_sum_obj, history_obj in reversed(zip(self._parsed[_FS_LAYERS_KEY],
|
||||
self._parsed[_HISTORY_KEY])):
|
||||
|
||||
try:
|
||||
image_digest = digest_tools.Digest.parse_digest(blob_sum_obj[_BLOB_SUM_KEY])
|
||||
except digest_tools.InvalidDigestException:
|
||||
raise ManifestInvalid()
|
||||
|
||||
metadata_string = history_obj[_V1_COMPAT_KEY]
|
||||
|
||||
v1_metadata = json.loads(metadata_string)
|
||||
command_list = v1_metadata.get('container_config', {}).get('Cmd', None)
|
||||
command = json.dumps(command_list) if command_list else None
|
||||
|
||||
extracted = ExtractedV1Metadata(v1_metadata['id'], v1_metadata.get('parent'),
|
||||
v1_metadata.get('created'), v1_metadata.get('comment'),
|
||||
command)
|
||||
yield ImageMetadata(image_digest, extracted, metadata_string)
|
||||
|
||||
@property
|
||||
def payload(self):
|
||||
protected = self._signatures[0][self.PROTECTED_KEY]
|
||||
parsed_protected = yaml.safe_load(jwt.utils.base64url_decode(protected))
|
||||
protected = str(self._signatures[0][_PROTECTED_KEY])
|
||||
|
||||
parsed_protected = json.loads(jwt.utils.base64url_decode(protected))
|
||||
logger.debug('parsed_protected: %s', parsed_protected)
|
||||
signed_content_head = self._bytes[:parsed_protected[self.FORMAT_LENGTH_KEY]]
|
||||
|
||||
signed_content_head = self._bytes[:parsed_protected[_FORMAT_LENGTH_KEY]]
|
||||
logger.debug('signed content head: %s', signed_content_head)
|
||||
signed_content_tail = jwt.utils.base64url_decode(parsed_protected[self.FORMAT_TAIL_KEY])
|
||||
|
||||
signed_content_tail = jwt.utils.base64url_decode(str(parsed_protected[_FORMAT_TAIL_KEY]))
|
||||
logger.debug('signed content tail: %s', signed_content_tail)
|
||||
return signed_content_head + signed_content_tail
|
||||
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/manifests/<regex("' + VALID_TAG_PATTERN + '"):tag_name>',
|
||||
methods=['GET'])
|
||||
class SignedManifestBuilder(object):
|
||||
""" Class which represents a manifest which is currently being built.
|
||||
"""
|
||||
def __init__(self, namespace, repo_name, tag, architecture='amd64', schema_ver=1):
|
||||
self._base_payload = {
|
||||
_REPO_TAG_KEY: tag,
|
||||
_REPO_NAME_KEY: '{0}/{1}'.format(namespace, repo_name),
|
||||
_ARCH_KEY: architecture,
|
||||
_SCHEMA_VER: schema_ver,
|
||||
}
|
||||
|
||||
self._fs_layer_digests = []
|
||||
self._history = []
|
||||
|
||||
def add_layer(self, layer_digest, v1_json_metadata):
|
||||
self._fs_layer_digests.append({
|
||||
_BLOB_SUM_KEY: layer_digest,
|
||||
})
|
||||
self._history.append({
|
||||
_V1_COMPAT_KEY: v1_json_metadata,
|
||||
})
|
||||
|
||||
def build(self, json_web_key):
|
||||
""" Build the payload and sign it, returning a SignedManifest object.
|
||||
"""
|
||||
payload = OrderedDict(self._base_payload)
|
||||
payload.update({
|
||||
_HISTORY_KEY: self._history,
|
||||
_FS_LAYERS_KEY: self._fs_layer_digests,
|
||||
})
|
||||
|
||||
payload_str = json.dumps(payload, indent=3)
|
||||
|
||||
split_point = payload_str.rfind('\n}')
|
||||
|
||||
protected_payload = {
|
||||
'formatTail': jwt.utils.base64url_encode(payload_str[split_point:]),
|
||||
'formatLength': split_point,
|
||||
'time': datetime.utcnow().strftime(ISO_DATETIME_FORMAT_ZULU),
|
||||
}
|
||||
protected = jwt.utils.base64url_encode(json.dumps(protected_payload))
|
||||
logger.debug('Generated protected block: %s', protected)
|
||||
|
||||
bytes_to_sign = '{0}.{1}'.format(protected, jwt.utils.base64url_encode(payload_str))
|
||||
|
||||
signer = SIGNER_ALGS[JWS_ALGORITHM]
|
||||
signature = jwt.utils.base64url_encode(signer.sign(bytes_to_sign, json_web_key.get_key()))
|
||||
logger.debug('Generated signature: %s', signature)
|
||||
|
||||
public_members = set(json_web_key.public_members)
|
||||
public_key = {comp: value for comp, value in json_web_key.to_dict().items()
|
||||
if comp in public_members}
|
||||
|
||||
signature_block = {
|
||||
'header': {
|
||||
'jwk': public_key,
|
||||
'alg': JWS_ALGORITHM,
|
||||
},
|
||||
'signature': signature,
|
||||
_PROTECTED_KEY: protected,
|
||||
}
|
||||
|
||||
logger.debug('Encoded signature block: %s', json.dumps(signature_block))
|
||||
|
||||
payload.update({
|
||||
_SIGNATURES_KEY: [signature_block],
|
||||
})
|
||||
|
||||
return SignedManifest(json.dumps(payload, indent=3))
|
||||
|
||||
|
||||
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
|
||||
@process_jwt_auth
|
||||
@require_repo_read
|
||||
@anon_protect
|
||||
def fetch_manifest_by_tagname(namespace, repo_name, tag_name):
|
||||
logger.debug('Fetching tag manifest with name: %s', tag_name)
|
||||
return make_response('Manifest {0}'.format(tag_name))
|
||||
def fetch_manifest_by_tagname(namespace, repo_name, manifest_ref):
|
||||
try:
|
||||
manifest = model.tag.load_tag_manifest(namespace, repo_name, manifest_ref)
|
||||
except model.InvalidManifestException:
|
||||
try:
|
||||
manifest = _generate_and_store_manifest(namespace, repo_name, manifest_ref)
|
||||
except model.DataModelException:
|
||||
logger.exception('Exception when generating manifest for %s/%s:%s', namespace, repo_name,
|
||||
manifest_ref)
|
||||
raise ManifestUnknown()
|
||||
|
||||
return make_response(manifest.json_data, 200)
|
||||
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/manifests/<regex("' + VALID_TAG_PATTERN + '"):tag_name>',
|
||||
methods=['PUT'])
|
||||
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
|
||||
@process_jwt_auth
|
||||
@require_repo_read
|
||||
@anon_protect
|
||||
def fetch_manifest_by_digest(namespace, repo_name, manifest_ref):
|
||||
try:
|
||||
manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
|
||||
except model.InvalidManifestException:
|
||||
# Without a tag name to reference, we can't make an attempt to generate the manifest
|
||||
raise ManifestUnknown()
|
||||
|
||||
return make_response(manifest.json_data, 200)
|
||||
|
||||
|
||||
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
|
||||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def write_manifest_by_tagname(namespace, repo_name, tag_name):
|
||||
manifest = SignedManifest(request.data)
|
||||
manifest_digest = digest_tools.sha256_digest(manifest.payload)
|
||||
def write_manifest_by_tagname(namespace, repo_name, manifest_ref):
|
||||
try:
|
||||
manifest = SignedManifest(request.data)
|
||||
except ValueError:
|
||||
raise ManifestInvalid()
|
||||
|
||||
if manifest.tag != manifest_ref:
|
||||
raise TagInvalid()
|
||||
|
||||
return _write_manifest(namespace, repo_name, manifest)
|
||||
|
||||
|
||||
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
|
||||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def write_manifest_by_digest(namespace, repo_name, manifest_ref):
|
||||
try:
|
||||
manifest = SignedManifest(request.data)
|
||||
except ValueError:
|
||||
raise ManifestInvalid()
|
||||
|
||||
if manifest.digest != manifest_ref:
|
||||
raise ManifestInvalid()
|
||||
|
||||
return _write_manifest(namespace, repo_name, manifest)
|
||||
|
||||
|
||||
def _write_manifest(namespace, repo_name, manifest):
|
||||
# Ensure that the manifest is for this repository.
|
||||
if manifest.namespace != namespace or manifest.repo_name != repo_name:
|
||||
raise NameInvalid()
|
||||
|
||||
# Ensure that the repository exists.
|
||||
repo = model.repository.get_repository(namespace, repo_name)
|
||||
if repo is None:
|
||||
raise NameInvalid()
|
||||
|
||||
# Lookup all the images and their parent images (if any) inside the manifest. This will let us
|
||||
# know which V1 images we need to synthesize and which ones are invalid.
|
||||
layers = list(manifest.layers)
|
||||
|
||||
docker_image_ids = [mdata.v1_metadata.docker_id for mdata in layers]
|
||||
parent_image_ids = [mdata.v1_metadata.parent for mdata in layers
|
||||
if mdata.v1_metadata.parent]
|
||||
all_image_ids = list(set(docker_image_ids + parent_image_ids))
|
||||
|
||||
images_query = model.image.lookup_repository_images(repo, all_image_ids)
|
||||
images_map = {image.docker_image_id: image for image in images_query}
|
||||
|
||||
# Lookup the storages associated with each blob in the manifest.
|
||||
checksums = [str(mdata.digest) for mdata in manifest.layers]
|
||||
storage_query = model.storage.lookup_repo_storages_by_content_checksum(repo, checksums)
|
||||
storage_map = {storage.content_checksum: storage for storage in storage_query}
|
||||
|
||||
# Synthesize the V1 metadata for each layer.
|
||||
manifest_digest = manifest.digest
|
||||
tag_name = manifest.tag
|
||||
|
||||
for mdata in layers:
|
||||
digest_str = str(mdata.digest)
|
||||
v1_mdata = mdata.v1_metadata
|
||||
|
||||
# If there is already a V1 image for this layer, nothing more to do.
|
||||
if v1_mdata.docker_id in images_map:
|
||||
continue
|
||||
|
||||
# Lookup the parent image for the layer, if any.
|
||||
parent_image = None
|
||||
if v1_mdata.parent is not None:
|
||||
parent_image = images_map.get(v1_mdata.parent)
|
||||
if parent_image is None:
|
||||
msg = 'Parent not found with docker image id {0}'.format(v1_mdata.parent)
|
||||
raise ManifestInvalid(detail={'message': msg})
|
||||
|
||||
# Synthesize and store the v1 metadata in the db.
|
||||
blob_storage = storage_map.get(digest_str)
|
||||
if blob_storage is None:
|
||||
raise BlobUnknown(detail={'digest': digest_str})
|
||||
|
||||
image = model.image.synthesize_v1_image(repo, blob_storage, v1_mdata.docker_id,
|
||||
v1_mdata.created, v1_mdata.comment, v1_mdata.command,
|
||||
mdata.v1_metadata_str, parent_image)
|
||||
|
||||
images_map[v1_mdata.docker_id] = image
|
||||
|
||||
if not layers:
|
||||
# The manifest doesn't actually reference any layers!
|
||||
raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})
|
||||
|
||||
# Store the manifest pointing to the tag.
|
||||
leaf_layer = layers[-1]
|
||||
model.tag.store_tag_manifest(namespace, repo_name, tag_name, leaf_layer.v1_metadata.docker_id,
|
||||
manifest_digest, request.data)
|
||||
|
||||
# Spawn the repo_push event.
|
||||
event_data = {
|
||||
'updated_tags': [tag_name],
|
||||
}
|
||||
|
||||
track_and_log('push_repo', repo)
|
||||
spawn_notification(repo, 'repo_push', event_data)
|
||||
|
||||
response = make_response('OK', 202)
|
||||
response.headers['Docker-Content-Digest'] = manifest_digest
|
||||
response.headers['Location'] = 'https://fun.com'
|
||||
response.headers['Location'] = url_for('v2.fetch_manifest_by_digest', namespace=namespace,
|
||||
repo_name=repo_name, manifest_ref=manifest_digest)
|
||||
return response
|
||||
|
||||
|
||||
# @v2_bp.route('/<namespace>/<repo_name>/manifests/<regex("' + digest_tools.DIGEST_PATTERN + '"):tag_digest>',
|
||||
# methods=['PUT'])
|
||||
# @process_jwt_auth
|
||||
# @require_repo_write
|
||||
# @anon_protect
|
||||
# def write_manifest(namespace, repo_name, tag_digest):
|
||||
# logger.debug('Writing tag manifest with name: %s', tag_digest)
|
||||
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
|
||||
@process_jwt_auth
|
||||
@require_repo_write
|
||||
@anon_protect
|
||||
def delete_manifest_by_digest(namespace, repo_name, manifest_ref):
|
||||
""" Delete the manifest specified by the digest. Note: there is no equivalent
|
||||
method for deleting by tag name because it is forbidden by the spec.
|
||||
"""
|
||||
try:
|
||||
manifest = model.tag.load_manifest_by_digest(namespace, repo_name, manifest_ref)
|
||||
except model.InvalidManifestException:
|
||||
# Without a tag name to reference, we can't make an attempt to generate the manifest
|
||||
raise ManifestUnknown()
|
||||
|
||||
# manifest_path = digest_tools.content_path(tag_digest)
|
||||
# storage.stream_write('local_us', manifest_path, get_input_stream(request))
|
||||
manifest.delete_instance()
|
||||
|
||||
# return make_response('Manifest {0}'.format(tag_digest))
|
||||
return make_response('', 202)
|
||||
|
||||
|
||||
def _generate_and_store_manifest(namespace, repo_name, tag_name):
|
||||
# First look up the tag object and its ancestors
|
||||
image = model.tag.get_tag_image(namespace, repo_name, tag_name)
|
||||
parents = model.image.get_parent_images(namespace, repo_name, image)
|
||||
|
||||
# Create and populate the manifest builder
|
||||
builder = SignedManifestBuilder(namespace, repo_name, tag_name)
|
||||
|
||||
# Add the leaf layer
|
||||
builder.add_layer(image.storage.content_checksum, image.v1_json_metadata)
|
||||
|
||||
for parent in parents:
|
||||
builder.add_layer(parent.storage.content_checksum, parent.v1_json_metadata)
|
||||
|
||||
# Sign the manifest with our signing key.
|
||||
manifest = builder.build(docker_v2_signing_key)
|
||||
manifest_row = model.tag.associate_generated_tag_manifest(namespace, repo_name, tag_name,
|
||||
manifest.digest, manifest.bytes)
|
||||
|
||||
return manifest_row
|
||||
|
|
|
|||
32
endpoints/v2/tag.py
Normal file
32
endpoints/v2/tag.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
from flask import jsonify, url_for
|
||||
|
||||
from endpoints.v2 import v2_bp, require_repo_read
|
||||
from endpoints.v2.errors import NameUnknown
|
||||
from endpoints.v2.v2util import add_pagination
|
||||
from auth.jwt_auth import process_jwt_auth
|
||||
from endpoints.decorators import anon_protect
|
||||
from data import model
|
||||
|
||||
@v2_bp.route('/<namespace>/<repo_name>/tags/list', methods=['GET'])
|
||||
@process_jwt_auth
|
||||
@require_repo_read
|
||||
@anon_protect
|
||||
def list_all_tags(namespace, repo_name):
|
||||
repository = model.repository.get_repository(namespace, repo_name)
|
||||
if repository is None:
|
||||
raise NameUnknown()
|
||||
|
||||
query = model.tag.list_repository_tags(namespace, repo_name)
|
||||
|
||||
url = url_for('v2.list_all_tags', namespace=namespace, repo_name=repo_name)
|
||||
link, query = add_pagination(query, url)
|
||||
|
||||
response = jsonify({
|
||||
'name': '{0}/{1}'.format(namespace, repo_name),
|
||||
'tags': [tag.name for tag in query],
|
||||
})
|
||||
|
||||
if link is not None:
|
||||
response.headers['Link'] = link
|
||||
|
||||
return response
|
||||
|
|
@ -1,6 +1,3 @@
|
|||
# XXX This code is not yet ready to be run in production, and should remain disabled until such
|
||||
# XXX time as this notice is removed.
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
|
|
@ -17,15 +14,17 @@ from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermissi
|
|||
CreateRepositoryPermission)
|
||||
from endpoints.v2 import v2_bp
|
||||
from util.cache import no_cache
|
||||
from util.names import parse_namespace_repository
|
||||
|
||||
from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX
|
||||
from endpoints.decorators import anon_protect
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour
|
||||
SCOPE_REGEX = re.compile(
|
||||
r'repository:([\.a-zA-Z0-9_\-]+/[\.a-zA-Z0-9_\-]+):(((push|pull|\*),)*(push|pull|\*))'
|
||||
r'^repository:([\.a-zA-Z0-9_\-]+/[\.a-zA-Z0-9_\-]+):(((push|pull|\*),)*(push|pull|\*))$'
|
||||
)
|
||||
ANONYMOUS_SUB = '(anonymous)'
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
|
|
@ -43,6 +42,7 @@ def load_private_key(private_key_file_path):
|
|||
@v2_bp.route('/auth')
|
||||
@process_auth
|
||||
@no_cache
|
||||
@anon_protect
|
||||
def generate_registry_jwt():
|
||||
""" This endpoint will generate a JWT conforming to the Docker registry v2 auth spec:
|
||||
https://docs.docker.com/registry/spec/auth/token/
|
||||
|
|
@ -54,13 +54,11 @@ def generate_registry_jwt():
|
|||
logger.debug('Scope request: %s', scope_param)
|
||||
|
||||
user = get_authenticated_user()
|
||||
|
||||
access = []
|
||||
if scope_param is not None:
|
||||
match = SCOPE_REGEX.match(scope_param)
|
||||
if match is None or match.end() != len(scope_param):
|
||||
if match is None:
|
||||
logger.debug('Match: %s', match)
|
||||
logger.debug('End: %s', match.end())
|
||||
logger.debug('len: %s', len(scope_param))
|
||||
logger.warning('Unable to decode repository and actions: %s', scope_param)
|
||||
abort(400)
|
||||
|
|
@ -71,21 +69,29 @@ def generate_registry_jwt():
|
|||
actions = match.group(2).split(',')
|
||||
|
||||
namespace, reponame = parse_namespace_repository(namespace_and_repo)
|
||||
|
||||
# Ensure that we are never creating an invalid repository.
|
||||
if not REPOSITORY_NAME_REGEX.match(reponame):
|
||||
abort(400)
|
||||
|
||||
if 'pull' in actions and 'push' in actions:
|
||||
if user is None:
|
||||
abort(401)
|
||||
|
||||
repo = model.repository.get_repository(namespace, reponame)
|
||||
if repo:
|
||||
if not ModifyRepositoryPermission(namespace, reponame):
|
||||
if not ModifyRepositoryPermission(namespace, reponame).can():
|
||||
abort(403)
|
||||
else:
|
||||
if not CreateRepositoryPermission(namespace):
|
||||
if not CreateRepositoryPermission(namespace).can():
|
||||
abort(403)
|
||||
logger.debug('Creating repository: %s/%s', namespace, reponame)
|
||||
model.repository.create_repository(namespace, reponame, user)
|
||||
elif 'pull' in actions:
|
||||
if not ReadRepositoryPermission(namespace, reponame):
|
||||
if (not ReadRepositoryPermission(namespace, reponame).can() and
|
||||
not model.repository.repository_is_public(namespace, reponame)):
|
||||
abort(403)
|
||||
|
||||
|
||||
access.append({
|
||||
'type': 'repository',
|
||||
'name': namespace_and_repo,
|
||||
|
|
@ -93,11 +99,12 @@ def generate_registry_jwt():
|
|||
})
|
||||
|
||||
token_data = {
|
||||
'iss': 'token-issuer',
|
||||
'iss': app.config['JWT_AUTH_TOKEN_ISSUER'],
|
||||
'aud': audience_param,
|
||||
'nbf': int(time.time()),
|
||||
'exp': int(time.time() + 60),
|
||||
'sub': user.username,
|
||||
'iat': int(time.time()),
|
||||
'exp': int(time.time() + TOKEN_VALIDITY_LIFETIME_S),
|
||||
'sub': user.username if user else ANONYMOUS_SUB,
|
||||
'access': access,
|
||||
}
|
||||
|
||||
|
|
|
|||
19
endpoints/v2/v2util.py
Normal file
19
endpoints/v2/v2util.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
from flask import request
|
||||
from app import get_app_url
|
||||
|
||||
_MAX_RESULTS_PER_PAGE = 100
|
||||
|
||||
def add_pagination(query, url):
|
||||
""" Adds optional pagination to the given query by looking for the Docker V2 pagination request
|
||||
args. """
|
||||
limit = request.args.get('n', None)
|
||||
page = request.args.get('page', 1)
|
||||
|
||||
if limit is None:
|
||||
return None, query
|
||||
|
||||
limit = max(limit, _MAX_RESULTS_PER_PAGE)
|
||||
url = get_app_url() + url
|
||||
query = query.paginate(page, limit)
|
||||
link = url + '?n=%s&last=%s; rel="next"' % (limit, page + 1)
|
||||
return link, query
|
||||
|
|
@ -29,11 +29,7 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag
|
|||
# the database.
|
||||
with database.UseThenDisconnect(app.config):
|
||||
image_list = list(model.image.get_parent_images(namespace, repository, repo_image))
|
||||
image_list.append(repo_image)
|
||||
|
||||
# Note: The image list ordering must be from top-level image, downward, so we reverse the order
|
||||
# here.
|
||||
image_list.reverse()
|
||||
image_list.insert(0, repo_image)
|
||||
|
||||
def get_next_image():
|
||||
for current_image in image_list:
|
||||
|
|
@ -41,7 +37,7 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag
|
|||
|
||||
def get_next_layer():
|
||||
for current_image_entry in image_list:
|
||||
current_image_path = store.image_layer_path(current_image_entry.storage.uuid)
|
||||
current_image_path = model.storage.get_layer_path(current_image_entry.storage)
|
||||
current_image_stream = store.stream_read_file(current_image_entry.storage.locations,
|
||||
current_image_path)
|
||||
|
||||
|
|
@ -51,7 +47,7 @@ def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, imag
|
|||
yield current_image_stream
|
||||
|
||||
stream = formatter.build_stream(namespace, repository, tag, synthetic_image_id, image_json,
|
||||
get_next_image, get_next_layer)
|
||||
get_next_image, get_next_layer, get_image_json)
|
||||
|
||||
return stream.read
|
||||
|
||||
|
|
@ -90,10 +86,14 @@ def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_location
|
|||
|
||||
queue_file.add_exception_handler(handle_exception)
|
||||
|
||||
image_path = store.image_layer_path(linked_storage_uuid)
|
||||
print "Starting write of sythn image"
|
||||
|
||||
image_path = store.v1_image_layer_path(linked_storage_uuid)
|
||||
store.stream_write(linked_locations, image_path, queue_file)
|
||||
queue_file.close()
|
||||
|
||||
print "Done writing synth image"
|
||||
|
||||
if not queue_file.raised_exception:
|
||||
# Setup the database (since this is a new process) and then disconnect immediately
|
||||
# once the operation completes.
|
||||
|
|
@ -170,9 +170,9 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
|
|||
derived = model.storage.find_or_create_derived_storage(repo_image.storage, verb,
|
||||
store.preferred_locations[0])
|
||||
|
||||
if not derived.uploading:
|
||||
if not derived.uploading and False:
|
||||
logger.debug('Derived %s image %s exists in storage', verb, derived.uuid)
|
||||
derived_layer_path = store.image_layer_path(derived.uuid)
|
||||
derived_layer_path = model.storage.get_layer_path(derived)
|
||||
download_url = store.get_direct_download_url(derived.locations, derived_layer_path)
|
||||
if download_url:
|
||||
logger.debug('Redirecting to download URL for derived %s image %s', verb, derived.uuid)
|
||||
|
|
@ -184,8 +184,8 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
|
|||
logger.debug('Sending cached derived %s image %s', verb, derived.uuid)
|
||||
return send_file(store.stream_read_file(derived.locations, derived_layer_path))
|
||||
|
||||
# Load the ancestry for the image.
|
||||
uuid = repo_image.storage.uuid
|
||||
# Load the full image list for the image.
|
||||
full_image_list = model.image.get_image_layers(repo_image)
|
||||
|
||||
logger.debug('Building and returning derived %s image %s', verb, derived.uuid)
|
||||
|
||||
|
|
|
|||
Reference in a new issue