vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'perf'

Browse source
This commit is contained in:
Joseph Schorr 2014-11-10 21:52:27 -05:00
commit e7cbda86f7
7 changed files with 219 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

4
endpoints/api/image.py
View file

@ -73,7 +73,7 @@ class RepositoryImage(RepositoryParamResource):
@nickname('getImage')
def get(self, namespace, repository, image_id):
""" Get the information available for the specified image. """
image = model.get_repo_image(namespace, repository, image_id)
image = model.get_repo_image_extended(namespace, repository, image_id)
if not image:
raise NotFound()
@ -94,7 +94,7 @@ class RepositoryImageChanges(RepositoryParamResource):
@nickname('getImageChanges')
def get(self, namespace, repository, image_id):
""" Get the list of changes for the specified image. """
image = model.get_repo_image(namespace, repository, image_id)
image = model.get_repo_image_extended(namespace, repository, image_id)
if not image:
raise NotFound()

20
endpoints/index.py
View file

@ -223,13 +223,20 @@ def create_repository(namespace, repository):
repo = model.create_repository(namespace, repository,
get_authenticated_user())
profile.debug('Determining added images')
added_images = OrderedDict([(desc['id'], desc)
for desc in image_descriptions])
profile.debug('Determining already added images')
added_images = OrderedDict([(desc['id'], desc) for desc in image_descriptions])
new_repo_images = dict(added_images)
for existing in model.get_repository_images(namespace, repository):
if existing.docker_image_id in new_repo_images:
# Optimization: Lookup any existing images in the repository with matching docker IDs and
# remove them from the added dict, so we don't need to look them up one-by-one.
def chunks(l, n):
for i in xrange(0, len(l), n):
yield l[i:i+n]
# Note: We do this in chunks in an effort to not hit the SQL query size limit.
for chunk in chunks(new_repo_images.keys(), 50):
existing_images = model.lookup_repository_images(namespace, repository, chunk)
for existing in existing_images:
added_images.pop(existing.docker_image_id)
profile.debug('Creating/Linking necessary images')
@ -241,9 +248,8 @@ def create_repository(namespace, repository):
profile.debug('Created images')
response = make_response('Created', 201)
track_and_log('push_repo', repo)
return response
return make_response('Created', 201)
@index.route('/repositories/<path:repository>/images', methods=['PUT'])

31
endpoints/registry.py
View file

@ -7,13 +7,13 @@ from functools import wraps
from datetime import datetime
from time import time
from app import storage as store, image_diff_queue
from app import storage as store, image_diff_queue, app
from auth.auth import process_auth, extract_namespace_repo_from_session
from util import checksums, changes
from util.http import abort, exact_abort
from auth.permissions import (ReadRepositoryPermission,
ModifyRepositoryPermission)
from data import model
from data import model, database
from util import gzipstream
@ -59,7 +59,7 @@ def require_completion(f):
@wraps(f)
def wrapper(namespace, repository, *args, **kwargs):
image_id = kwargs['image_id']
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if image_is_uploading(repo_image):
abort(400, 'Image %(image_id)s is being uploaded, retry later',
issue='upload-in-progress', image_id=kwargs['image_id'])
@ -103,7 +103,7 @@ def head_image_layer(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
if permission.can() or model.repository_is_public(namespace, repository):
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
profile.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
@ -136,7 +136,7 @@ def get_image_layer(namespace, repository, image_id, headers):
profile.debug('Checking repo permissions')
if permission.can() or model.repository_is_public(namespace, repository):
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up the layer path')
try:
@ -151,6 +151,10 @@ def get_image_layer(namespace, repository, image_id, headers):
return resp
profile.debug('Streaming layer data')
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
return Response(store.stream_read(repo_image.storage.locations, path), headers=headers)
except (IOError, AttributeError):
profile.debug('Image not found')
@ -170,7 +174,7 @@ def put_image_layer(namespace, repository, image_id):
abort(403)
profile.debug('Retrieving image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
try:
profile.debug('Retrieving image data')
uuid = repo_image.storage.uuid
@ -213,7 +217,8 @@ def put_image_layer(namespace, repository, image_id):
sr.add_handler(sum_hndlr)
# Stream write the data to storage.
store.stream_write(repo_image.storage.locations, layer_path, sr)
with database.CloseForLongOperation(app.config):
store.stream_write(repo_image.storage.locations, layer_path, sr)
# Append the computed checksum.
csums = []
@ -293,7 +298,7 @@ def put_image_checksum(namespace, repository, image_id):
issue='missing-checksum-cookie', image_id=image_id)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image or not repo_image.storage:
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
@ -348,7 +353,7 @@ def get_image_json(namespace, repository, image_id, headers):
abort(403)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up repo layer data')
try:
@ -379,7 +384,7 @@ def get_image_ancestry(namespace, repository, image_id, headers):
abort(403)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
profile.debug('Looking up image data')
try:
@ -443,7 +448,7 @@ def put_image_json(namespace, repository, image_id):
issue='invalid-request', image_id=image_id)
profile.debug('Looking up repo image')
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
profile.debug('Image not found')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
@ -460,7 +465,7 @@ def put_image_json(namespace, repository, image_id):
parent_image = None
if parent_id:
profile.debug('Looking up parent image')
parent_image = model.get_repo_image(namespace, repository, parent_id)
parent_image = model.get_repo_image_extended(namespace, repository, parent_id)
parent_uuid = parent_image and parent_image.storage.uuid
parent_locations = parent_image and parent_image.storage.locations
@ -513,7 +518,7 @@ def put_image_json(namespace, repository, image_id):
def process_image_changes(namespace, repository, image_id):
logger.debug('Generating diffs for image: %s' % image_id)
repo_image = model.get_repo_image(namespace, repository, image_id)
repo_image = model.get_repo_image_extended(namespace, repository, image_id)
if not repo_image:
logger.warning('No image for id: %s', image_id)
return None, None

36
endpoints/verbs.py
View file

@ -21,24 +21,28 @@ from util.dockerloadformat import build_docker_load_stream
verbs = Blueprint('verbs', __name__)
logger = logging.getLogger(__name__)
def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_list):
def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_id_list):
store = Storage(app)
# For performance reasons, we load the full image list here, cache it, then disconnect from
# the database.
with database.UseThenDisconnect(app.config):
image_list = model.get_matching_repository_images(namespace, repository, image_id_list)
def get_next_image():
for current_image_id in image_list:
yield model.get_repo_image(namespace, repository, current_image_id)
for current_image in image_list:
yield current_image
def get_next_layer():
for current_image_id in image_list:
current_image_entry = model.get_repo_image(namespace, repository, current_image_id)
for current_image_entry in image_list:
current_image_path = store.image_layer_path(current_image_entry.storage.uuid)
current_image_stream = store.stream_read_file(current_image_entry.storage.locations,
current_image_path)
current_image_id = current_image_entry.id
logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
yield current_image_stream
database.configure(app.config)
stream = build_docker_load_stream(namespace, repository, tag, synthetic_image_id, image_json,
get_next_image, get_next_layer)
@ -46,12 +50,13 @@ def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, ima
def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, queue_file):
database.configure(app.config)
store = Storage(app)
def handle_exception(ex):
logger.debug('Exception when building squashed image %s: %s', linked_storage_uuid, ex)
model.delete_derived_storage_by_uuid(linked_storage_uuid)
with database.UseThenDisconnect(app.config):
model.delete_derived_storage_by_uuid(linked_storage_uuid)
queue_file.add_exception_handler(handle_exception)
@ -60,9 +65,10 @@ def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, que
queue_file.close()
if not queue_file.raised_exception:
done_uploading = model.get_storage_by_uuid(linked_storage_uuid)
done_uploading.uploading = False
done_uploading.save()
with database.UseThenDisconnect(app.config):
done_uploading = model.get_storage_by_uuid(linked_storage_uuid)
done_uploading.uploading = False
done_uploading.save()
@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
@ -77,7 +83,7 @@ def get_squashed_tag(namespace, repository, tag):
abort(404)
# Lookup the tag's image and storage.
repo_image = model.get_repo_image(namespace, repository, tag_image.docker_image_id)
repo_image = model.get_repo_image_extended(namespace, repository, tag_image.docker_image_id)
if not repo_image:
abort(404)
@ -95,6 +101,9 @@ def get_squashed_tag(namespace, repository, tag):
logger.debug('Redirecting to download URL for derived image %s', derived.uuid)
return redirect(download_url)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
logger.debug('Sending cached derived image %s', derived.uuid)
return send_file(store.stream_read_file(derived.locations, derived_layer_path))
@ -132,6 +141,9 @@ def get_squashed_tag(namespace, repository, tag):
storage_args = (derived.uuid, derived.locations, storage_queue_file)
QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup)
# Close the database handle here for this process before we send the long download.
database.close_db_filter(None)
# Return the client's data.
return send_file(client_queue_file)