4bf4ce33c9
This change replaces the metricqueue library with a native Prometheus client implementation with the intention to aggregated results with the Prometheus PushGateway. This change also adds instrumentation for greenlet context switches.
409 lines
15 KiB
Python
409 lines
15 KiB
Python
import logging
|
|
import json
|
|
|
|
from functools import wraps
|
|
from datetime import datetime
|
|
from time import time
|
|
|
|
from flask import make_response, request, session, Response, redirect, abort as flask_abort
|
|
|
|
from app import storage as store, app, docker_v2_signing_key
|
|
from auth.auth_context import get_authenticated_user
|
|
from auth.decorators import extract_namespace_repo_from_session, process_auth
|
|
from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission)
|
|
from data import database
|
|
from data.registry_model import registry_model
|
|
from data.registry_model.blobuploader import upload_blob, BlobUploadSettings, BlobUploadException
|
|
from data.registry_model.manifestbuilder import lookup_manifest_builder
|
|
from digest import checksums
|
|
from endpoints.metrics import image_pulled_bytes
|
|
from endpoints.v1 import v1_bp, check_v1_push_enabled
|
|
from endpoints.v1.index import ensure_namespace_enabled
|
|
from endpoints.decorators import (anon_protect, check_region_blacklisted, check_repository_state,
|
|
check_readonly)
|
|
from util.http import abort
|
|
from util.registry.replication import queue_storage_replication
|
|
from util.request import get_request_ip
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def require_completion(f):
|
|
""" This make sure that the image push correctly finished. """
|
|
@wraps(f)
|
|
def wrapper(namespace, repository, *args, **kwargs):
|
|
image_id = kwargs['image_id']
|
|
repository_ref = registry_model.lookup_repository(namespace, repository)
|
|
if repository_ref is not None:
|
|
legacy_image = registry_model.get_legacy_image(repository_ref, image_id)
|
|
if legacy_image is not None and legacy_image.uploading:
|
|
abort(400, 'Image %(image_id)s is being uploaded, retry later', issue='upload-in-progress',
|
|
image_id=image_id)
|
|
|
|
return f(namespace, repository, *args, **kwargs)
|
|
return wrapper
|
|
|
|
|
|
def set_cache_headers(f):
|
|
"""Returns HTTP headers suitable for caching."""
|
|
|
|
@wraps(f)
|
|
def wrapper(*args, **kwargs):
|
|
# Set TTL to 1 year by default
|
|
ttl = 31536000
|
|
expires = datetime.fromtimestamp(int(time()) + ttl)
|
|
expires = expires.strftime('%a, %d %b %Y %H:%M:%S GMT')
|
|
headers = {
|
|
'Cache-Control': 'public, max-age={0}'.format(ttl),
|
|
'Expires': expires,
|
|
'Last-Modified': 'Thu, 01 Jan 1970 00:00:00 GMT',}
|
|
if 'If-Modified-Since' in request.headers:
|
|
response = make_response('Not modified', 304)
|
|
response.headers.extend(headers)
|
|
return response
|
|
kwargs['headers'] = headers
|
|
# Prevent the Cookie to be sent when the object is cacheable
|
|
session.modified = False
|
|
return f(*args, **kwargs)
|
|
|
|
return wrapper
|
|
|
|
|
|
@v1_bp.route('/images/<image_id>/layer', methods=['HEAD'])
|
|
@process_auth
|
|
@extract_namespace_repo_from_session
|
|
@ensure_namespace_enabled
|
|
@require_completion
|
|
@set_cache_headers
|
|
@anon_protect
|
|
def head_image_layer(namespace, repository, image_id, headers):
|
|
permission = ReadRepositoryPermission(namespace, repository)
|
|
repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image')
|
|
|
|
logger.debug('Checking repo permissions')
|
|
if permission.can() or (repository_ref is not None and repository_ref.is_public):
|
|
if repository_ref is None:
|
|
abort(404)
|
|
|
|
logger.debug('Looking up placement locations')
|
|
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
|
|
if legacy_image is None:
|
|
logger.debug('Could not find any blob placement locations')
|
|
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
|
|
|
# Add the Accept-Ranges header if the storage engine supports resumable
|
|
# downloads.
|
|
extra_headers = {}
|
|
if store.get_supports_resumable_downloads(legacy_image.blob.placements):
|
|
logger.debug('Storage supports resumable downloads')
|
|
extra_headers['Accept-Ranges'] = 'bytes'
|
|
|
|
resp = make_response('')
|
|
resp.headers.extend(headers)
|
|
resp.headers.extend(extra_headers)
|
|
return resp
|
|
|
|
abort(403)
|
|
|
|
|
|
@v1_bp.route('/images/<image_id>/layer', methods=['GET'])
|
|
@process_auth
|
|
@extract_namespace_repo_from_session
|
|
@ensure_namespace_enabled
|
|
@require_completion
|
|
@set_cache_headers
|
|
@check_region_blacklisted()
|
|
@anon_protect
|
|
def get_image_layer(namespace, repository, image_id, headers):
|
|
permission = ReadRepositoryPermission(namespace, repository)
|
|
repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image')
|
|
|
|
logger.debug('Checking repo permissions')
|
|
if permission.can() or (repository_ref is not None and repository_ref.is_public):
|
|
if repository_ref is None:
|
|
abort(404)
|
|
|
|
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
|
|
if legacy_image is None:
|
|
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
|
|
|
path = legacy_image.blob.storage_path
|
|
image_pulled_bytes.labels('v1').inc(legacy_image.blob.compressed_size)
|
|
|
|
try:
|
|
logger.debug('Looking up the direct download URL for path: %s', path)
|
|
direct_download_url = store.get_direct_download_url(legacy_image.blob.placements, path,
|
|
get_request_ip())
|
|
if direct_download_url:
|
|
logger.debug('Returning direct download URL')
|
|
resp = redirect(direct_download_url)
|
|
return resp
|
|
|
|
# Close the database handle here for this process before we send the long download.
|
|
database.close_db_filter(None)
|
|
logger.debug('Streaming layer data')
|
|
return Response(store.stream_read(legacy_image.blob.placements, path), headers=headers)
|
|
except (IOError, AttributeError):
|
|
logger.exception('Image layer data not found')
|
|
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
|
|
|
abort(403)
|
|
|
|
|
|
@v1_bp.route('/images/<image_id>/layer', methods=['PUT'])
|
|
@process_auth
|
|
@extract_namespace_repo_from_session
|
|
@check_v1_push_enabled()
|
|
@ensure_namespace_enabled
|
|
@check_repository_state
|
|
@anon_protect
|
|
@check_readonly
|
|
def put_image_layer(namespace, repository, image_id):
|
|
logger.debug('Checking repo permissions')
|
|
permission = ModifyRepositoryPermission(namespace, repository)
|
|
if not permission.can():
|
|
abort(403)
|
|
|
|
repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image')
|
|
if repository_ref is None:
|
|
abort(403)
|
|
|
|
logger.debug('Checking for image in manifest builder')
|
|
builder = lookup_manifest_builder(repository_ref, session.get('manifest_builder'), store,
|
|
docker_v2_signing_key)
|
|
if builder is None:
|
|
abort(400)
|
|
|
|
layer = builder.lookup_layer(image_id)
|
|
if layer is None:
|
|
abort(404)
|
|
|
|
logger.debug('Storing layer data')
|
|
input_stream = request.stream
|
|
if request.headers.get('transfer-encoding') == 'chunked':
|
|
# Careful, might work only with WSGI servers supporting chunked
|
|
# encoding (Gunicorn)
|
|
input_stream = request.environ['wsgi.input']
|
|
|
|
expiration_sec = app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']
|
|
settings = BlobUploadSettings(maximum_blob_size=app.config['MAXIMUM_LAYER_SIZE'],
|
|
bittorrent_piece_size=app.config['BITTORRENT_PIECE_SIZE'],
|
|
committed_blob_expiration=expiration_sec)
|
|
|
|
extra_handlers = []
|
|
|
|
# Add a handler that copies the data into a temp file. This is used to calculate the tarsum,
|
|
# which is only needed for older versions of Docker.
|
|
requires_tarsum = bool(builder.get_layer_checksums(layer))
|
|
if requires_tarsum:
|
|
tmp, tmp_hndlr = store.temp_store_handler()
|
|
extra_handlers.append(tmp_hndlr)
|
|
|
|
# Add a handler which computes the simple Docker V1 checksum.
|
|
h, sum_hndlr = checksums.simple_checksum_handler(layer.v1_metadata_string)
|
|
extra_handlers.append(sum_hndlr)
|
|
|
|
uploaded_blob = None
|
|
try:
|
|
with upload_blob(repository_ref, store, settings,
|
|
extra_blob_stream_handlers=extra_handlers) as manager:
|
|
manager.upload_chunk(app.config, input_stream)
|
|
uploaded_blob = manager.commit_to_blob(app.config)
|
|
except BlobUploadException:
|
|
logger.exception('Exception when writing image data')
|
|
abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id)
|
|
|
|
# Compute the final checksum
|
|
csums = []
|
|
csums.append('sha256:{0}'.format(h.hexdigest()))
|
|
|
|
try:
|
|
if requires_tarsum:
|
|
tmp.seek(0)
|
|
csums.append(checksums.compute_tarsum(tmp, layer.v1_metadata_string))
|
|
tmp.close()
|
|
except (IOError, checksums.TarError) as exc:
|
|
logger.debug('put_image_layer: Error when computing tarsum %s', exc)
|
|
|
|
# If there was already a precomputed checksum, validate against it now.
|
|
if builder.get_layer_checksums(layer):
|
|
checksum = builder.get_layer_checksums(layer)[0]
|
|
if not builder.validate_layer_checksum(layer, checksum):
|
|
logger.debug('put_image_checksum: Wrong checksum. Given: %s and expected: %s', checksum,
|
|
builder.get_layer_checksums(layer))
|
|
abort(400, 'Checksum mismatch for image: %(image_id)s', issue='checksum-mismatch',
|
|
image_id=image_id)
|
|
|
|
# Assign the blob to the layer in the manifest.
|
|
if not builder.assign_layer_blob(layer, uploaded_blob, csums):
|
|
abort(500, 'Something went wrong')
|
|
|
|
# Send a job to the work queue to replicate the image layer.
|
|
# TODO: move this into a better place.
|
|
queue_storage_replication(namespace, uploaded_blob)
|
|
|
|
return make_response('true', 200)
|
|
|
|
|
|
@v1_bp.route('/images/<image_id>/checksum', methods=['PUT'])
|
|
@process_auth
|
|
@extract_namespace_repo_from_session
|
|
@check_v1_push_enabled()
|
|
@ensure_namespace_enabled
|
|
@check_repository_state
|
|
@anon_protect
|
|
@check_readonly
|
|
def put_image_checksum(namespace, repository, image_id):
|
|
logger.debug('Checking repo permissions')
|
|
permission = ModifyRepositoryPermission(namespace, repository)
|
|
if not permission.can():
|
|
abort(403)
|
|
|
|
repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image')
|
|
if repository_ref is None:
|
|
abort(403)
|
|
|
|
# Docker Version < 0.10 (tarsum+sha):
|
|
old_checksum = request.headers.get('X-Docker-Checksum')
|
|
|
|
# Docker Version >= 0.10 (sha):
|
|
new_checksum = request.headers.get('X-Docker-Checksum-Payload')
|
|
|
|
checksum = new_checksum or old_checksum
|
|
if not checksum:
|
|
abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum',
|
|
image_id=image_id)
|
|
|
|
logger.debug('Checking for image in manifest builder')
|
|
builder = lookup_manifest_builder(repository_ref, session.get('manifest_builder'), store,
|
|
docker_v2_signing_key)
|
|
if builder is None:
|
|
abort(400)
|
|
|
|
layer = builder.lookup_layer(image_id)
|
|
if layer is None:
|
|
abort(404)
|
|
|
|
if old_checksum:
|
|
builder.save_precomputed_checksum(layer, checksum)
|
|
return make_response('true', 200)
|
|
|
|
if not builder.validate_layer_checksum(layer, checksum):
|
|
logger.debug('put_image_checksum: Wrong checksum. Given: %s and expected: %s', checksum,
|
|
builder.get_layer_checksums(layer))
|
|
abort(400, 'Checksum mismatch for image: %(image_id)s', issue='checksum-mismatch',
|
|
image_id=image_id)
|
|
|
|
return make_response('true', 200)
|
|
|
|
|
|
@v1_bp.route('/images/<image_id>/json', methods=['GET'])
|
|
@process_auth
|
|
@extract_namespace_repo_from_session
|
|
@ensure_namespace_enabled
|
|
@require_completion
|
|
@set_cache_headers
|
|
@anon_protect
|
|
def get_image_json(namespace, repository, image_id, headers):
|
|
logger.debug('Checking repo permissions')
|
|
permission = ReadRepositoryPermission(namespace, repository)
|
|
repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image')
|
|
if not permission.can() and not (repository_ref is not None and repository_ref.is_public):
|
|
abort(403)
|
|
|
|
logger.debug('Looking up repo image')
|
|
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_blob=True)
|
|
if legacy_image is None:
|
|
flask_abort(404)
|
|
|
|
size = legacy_image.blob.compressed_size
|
|
if size is not None:
|
|
# Note: X-Docker-Size is optional and we *can* end up with a NULL image_size,
|
|
# so handle this case rather than failing.
|
|
headers['X-Docker-Size'] = str(size)
|
|
|
|
response = make_response(legacy_image.v1_metadata_string, 200)
|
|
response.headers.extend(headers)
|
|
return response
|
|
|
|
|
|
@v1_bp.route('/images/<image_id>/ancestry', methods=['GET'])
|
|
@process_auth
|
|
@extract_namespace_repo_from_session
|
|
@ensure_namespace_enabled
|
|
@require_completion
|
|
@set_cache_headers
|
|
@anon_protect
|
|
def get_image_ancestry(namespace, repository, image_id, headers):
|
|
logger.debug('Checking repo permissions')
|
|
permission = ReadRepositoryPermission(namespace, repository)
|
|
repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image')
|
|
if not permission.can() and not (repository_ref is not None and repository_ref.is_public):
|
|
abort(403)
|
|
|
|
logger.debug('Looking up repo image')
|
|
legacy_image = registry_model.get_legacy_image(repository_ref, image_id, include_parents=True)
|
|
if legacy_image is None:
|
|
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
|
|
|
# NOTE: We can not use jsonify here because we are returning a list not an object.
|
|
ancestor_ids = ([legacy_image.docker_image_id] +
|
|
[a.docker_image_id for a in legacy_image.parents])
|
|
response = make_response(json.dumps(ancestor_ids), 200)
|
|
response.headers.extend(headers)
|
|
return response
|
|
|
|
|
|
@v1_bp.route('/images/<image_id>/json', methods=['PUT'])
|
|
@process_auth
|
|
@extract_namespace_repo_from_session
|
|
@check_v1_push_enabled()
|
|
@ensure_namespace_enabled
|
|
@check_repository_state
|
|
@anon_protect
|
|
@check_readonly
|
|
def put_image_json(namespace, repository, image_id):
|
|
logger.debug('Checking repo permissions')
|
|
permission = ModifyRepositoryPermission(namespace, repository)
|
|
if not permission.can():
|
|
abort(403)
|
|
|
|
repository_ref = registry_model.lookup_repository(namespace, repository, kind_filter='image')
|
|
if repository_ref is None:
|
|
abort(403)
|
|
|
|
builder = lookup_manifest_builder(repository_ref, session.get('manifest_builder'), store,
|
|
docker_v2_signing_key)
|
|
if builder is None:
|
|
abort(400)
|
|
|
|
logger.debug('Parsing image JSON')
|
|
try:
|
|
uploaded_metadata = request.data
|
|
data = json.loads(uploaded_metadata.decode('utf8'))
|
|
except ValueError:
|
|
pass
|
|
|
|
if not data or not isinstance(data, dict):
|
|
abort(400, 'Invalid JSON for image: %(image_id)s\nJSON: %(json)s', issue='invalid-request',
|
|
image_id=image_id, json=request.data)
|
|
|
|
if 'id' not in data:
|
|
abort(400, 'Missing key `id` in JSON for image: %(image_id)s', issue='invalid-request',
|
|
image_id=image_id)
|
|
|
|
if image_id != data['id']:
|
|
abort(400, 'JSON data contains invalid id for image: %(image_id)s', issue='invalid-request',
|
|
image_id=image_id)
|
|
|
|
logger.debug('Looking up repo image')
|
|
location_pref = store.preferred_locations[0]
|
|
username = get_authenticated_user() and get_authenticated_user().username
|
|
layer = builder.start_layer(image_id, uploaded_metadata, location_pref, username,
|
|
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
|
|
if layer is None:
|
|
abort(400, 'Image %(image_id)s has invalid metadata',
|
|
issue='invalid-request', image_id=image_id)
|
|
|
|
return make_response('true', 200)
|