Add general+trackable logging to every request and add fine grain logging to the index and registry for determining possible performance regressions
This commit is contained in:
Joseph Schorr
parent
a5a61576ae
commit
4da49da730
3 changed files with 122 additions and 18 deletions
|
|
@ -21,7 +21,7 @@ from data import model
|
|||
registry = Blueprint('registry', __name__)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
profile = logging.getLogger('application.profiler')
|
||||
|
||||
class SocketReader(object):
|
||||
def __init__(self, fp):
|
||||
|
|
@ -88,17 +88,28 @@ def set_cache_headers(f):
|
|||
@set_cache_headers
|
||||
def get_image_layer(namespace, repository, image_id, headers):
|
||||
permission = ReadRepositoryPermission(namespace, repository)
|
||||
|
||||
profile.debug('Checking repo permissions')
|
||||
if permission.can() or model.repository_is_public(namespace, repository):
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up the layer path')
|
||||
path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
|
||||
profile.debug('Looking up the direct download URL')
|
||||
direct_download_url = store.get_direct_download_url(path)
|
||||
|
||||
if direct_download_url:
|
||||
profile.debug('Returning direct download URL')
|
||||
return redirect(direct_download_url)
|
||||
try:
|
||||
profile.debug('Streaming layer data')
|
||||
return Response(store.stream_read(path), headers=headers)
|
||||
except IOError:
|
||||
profile.debug('Image not found')
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
|
|
@ -109,25 +120,32 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
@process_auth
|
||||
@extract_namespace_repo_from_session
|
||||
def put_image_layer(namespace, repository, image_id):
|
||||
profile.debug('Checking repo permissions')
|
||||
permission = ModifyRepositoryPermission(namespace, repository)
|
||||
if not permission.can():
|
||||
abort(403)
|
||||
|
||||
profile.debug('Retrieving image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
try:
|
||||
profile.debug('Retrieving image data')
|
||||
json_data = store.get_content(store.image_json_path(namespace, repository,
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
profile.debug('Retrieving image path info')
|
||||
layer_path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
|
||||
if store.exists(layer_path) and not store.exists(mark_path):
|
||||
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
|
||||
|
||||
profile.debug('Storing layer data')
|
||||
|
||||
input_stream = request.stream
|
||||
if request.headers.get('transfer-encoding') == 'chunked':
|
||||
# Careful, might work only with WSGI servers supporting chunked
|
||||
|
|
@ -178,7 +196,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
|
||||
# The layer is ready for download, send a job to the work queue to
|
||||
# process it.
|
||||
logger.debug('Queing diffs job for image: %s' % image_id)
|
||||
profile.debug('Adding layer to diff queue')
|
||||
image_diff_queue.put([namespace, repository, image_id], json.dumps({
|
||||
'namespace': namespace,
|
||||
'repository': repository,
|
||||
|
|
@ -192,6 +210,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
@process_auth
|
||||
@extract_namespace_repo_from_session
|
||||
def put_image_checksum(namespace, repository, image_id):
|
||||
profile.debug('Checking repo permissions')
|
||||
permission = ModifyRepositoryPermission(namespace, repository)
|
||||
if not permission.can():
|
||||
abort(403)
|
||||
|
|
@ -204,17 +223,23 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
abort(400, 'Checksum not found in Cookie for image %(imaage_id)s',
|
||||
issue='missing-checksum-cookie', image_id=image_id)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up repo layer data')
|
||||
if not store.exists(store.image_json_path(namespace, repository, image_id,
|
||||
uuid)):
|
||||
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
|
||||
|
||||
profile.debug('Marking image path')
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
if not store.exists(mark_path):
|
||||
abort(409, 'Cannot set checksum for image %(image_id)s',
|
||||
issue='image-write-error', image_id=image_id)
|
||||
|
||||
profile.debug('Storing image checksum')
|
||||
err = store_checksum(namespace, repository, image_id, uuid, checksum)
|
||||
if err:
|
||||
abort(400, err)
|
||||
|
|
@ -231,7 +256,7 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
|
||||
# The layer is ready for download, send a job to the work queue to
|
||||
# process it.
|
||||
logger.debug('Queing diffs job for image: %s' % image_id)
|
||||
profile.debug('Adding layer to diff queue')
|
||||
image_diff_queue.put([namespace, repository, image_id], json.dumps({
|
||||
'namespace': namespace,
|
||||
'repository': repository,
|
||||
|
|
@ -247,20 +272,24 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
@require_completion
|
||||
@set_cache_headers
|
||||
def get_image_json(namespace, repository, image_id, headers):
|
||||
profile.debug('Checking repo permissions')
|
||||
permission = ReadRepositoryPermission(namespace, repository)
|
||||
if not permission.can() and not model.repository_is_public(namespace,
|
||||
repository):
|
||||
abort(403)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up repo layer data')
|
||||
try:
|
||||
data = store.get_content(store.image_json_path(namespace, repository,
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
flask_abort(404)
|
||||
|
||||
profile.debug('Looking up repo layer size')
|
||||
try:
|
||||
size = store.get_size(store.image_layer_path(namespace, repository,
|
||||
image_id, uuid))
|
||||
|
|
@ -268,6 +297,7 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
except OSError:
|
||||
pass
|
||||
|
||||
profile.debug('Retrieving checksum')
|
||||
checksum_path = store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid)
|
||||
if store.exists(checksum_path):
|
||||
|
|
@ -284,14 +314,17 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
@require_completion
|
||||
@set_cache_headers
|
||||
def get_image_ancestry(namespace, repository, image_id, headers):
|
||||
profile.debug('Checking repo permissions')
|
||||
permission = ReadRepositoryPermission(namespace, repository)
|
||||
if not permission.can() and not model.repository_is_public(namespace,
|
||||
repository):
|
||||
abort(403)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up image data')
|
||||
try:
|
||||
data = store.get_content(store.image_ancestry_path(namespace, repository,
|
||||
image_id, uuid))
|
||||
|
|
@ -299,8 +332,11 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
profile.debug('Converting to <-> from JSON')
|
||||
response = make_response(json.dumps(json.loads(data)), 200)
|
||||
response.headers.extend(headers)
|
||||
|
||||
profile.debug('Done')
|
||||
return response
|
||||
|
||||
|
||||
|
|
@ -335,10 +371,12 @@ def store_checksum(namespace, repository, image_id, uuid, checksum):
|
|||
@process_auth
|
||||
@extract_namespace_repo_from_session
|
||||
def put_image_json(namespace, repository, image_id):
|
||||
profile.debug('Checking repo permissions')
|
||||
permission = ModifyRepositoryPermission(namespace, repository)
|
||||
if not permission.can():
|
||||
abort(403)
|
||||
|
||||
profile.debug('Parsing image JSON')
|
||||
try:
|
||||
data = json.loads(request.data)
|
||||
except json.JSONDecodeError:
|
||||
|
|
@ -351,6 +389,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
abort(400, 'Missing key `id` in JSON for image: %(image_id)s',
|
||||
issue='invalid-request', image_id=image_id)
|
||||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
|
|
@ -358,12 +397,14 @@ def put_image_json(namespace, repository, image_id):
|
|||
checksum = request.headers.get('X-Docker-Checksum')
|
||||
if checksum:
|
||||
# Storing the checksum is optional at this stage
|
||||
profile.debug('Storing image checksum')
|
||||
err = store_checksum(namespace, repository, image_id, uuid, checksum)
|
||||
if err:
|
||||
abort(400, err, issue='write-error')
|
||||
|
||||
else:
|
||||
# We cleanup any old checksum in case it's a retry after a fail
|
||||
profile.debug('Cleanup old checksum')
|
||||
store.remove(store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid))
|
||||
if image_id != data['id']:
|
||||
|
|
@ -374,18 +415,26 @@ def put_image_json(namespace, repository, image_id):
|
|||
|
||||
parent_image = None
|
||||
if parent_id:
|
||||
profile.debug('Looking up parent image')
|
||||
parent_image = model.get_repo_image(namespace, repository, parent_id)
|
||||
|
||||
parent_uuid = (parent_image and parent_image.storage and
|
||||
parent_image.storage.uuid)
|
||||
|
||||
if parent_id:
|
||||
profile.debug('Looking up parent image data')
|
||||
|
||||
if (parent_id and not
|
||||
store.exists(store.image_json_path(namespace, repository, parent_id,
|
||||
parent_uuid))):
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
|
||||
|
||||
profile.debug('Looking up image storage paths')
|
||||
json_path = store.image_json_path(namespace, repository, image_id, uuid)
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
|
||||
profile.debug('Checking if image already exists')
|
||||
if store.exists(json_path) and not store.exists(mark_path):
|
||||
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
|
||||
|
||||
|
|
@ -394,13 +443,23 @@ def put_image_json(namespace, repository, image_id):
|
|||
# save the metadata
|
||||
command_list = data.get('container_config', {}).get('Cmd', None)
|
||||
command = json.dumps(command_list) if command_list else None
|
||||
|
||||
profile.debug('Setting image metadata')
|
||||
model.set_image_metadata(image_id, namespace, repository,
|
||||
data.get('created'), data.get('comment'), command,
|
||||
parent_image)
|
||||
|
||||
profile.debug('Putting mark path')
|
||||
store.put_content(mark_path, 'true')
|
||||
|
||||
profile.debug('Putting json path')
|
||||
store.put_content(json_path, request.data)
|
||||
|
||||
profile.debug('Generating image ancestry')
|
||||
generate_ancestry(namespace, repository, image_id, uuid, parent_id,
|
||||
parent_uuid)
|
||||
|
||||
profile.debug('Done')
|
||||
return make_response('true', 200)
|
||||
|
||||
|
||||
|
|
|
|||
Reference in a new issue