vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Have the squashing system write the data (via a double queue system and multiprocessing) to both the client and the stack's storage. On subsequent calls, if the synthetic image exists, it will be returned directly instead of being recomputed

Browse source
This commit is contained in:
Joseph Schorr 2014-09-16 22:43:19 -04:00
parent 5cca609c55
commit 1cfb6fc353
3 changed files with 146 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

63
endpoints/verbs.py
View file

@ -11,6 +11,8 @@ from auth.permissions import ReadRepositoryPermission
from data import model
from endpoints.registry import set_cache_headers
from util.queuefile import QueueFile
from util.queueprocess import QueueProcess
from util.gzipwrap import GzipWrap
from util.streamlayerformat import StreamLayerMerger
@ -19,6 +21,24 @@ from werkzeug.wsgi import wrap_file
verbs = Blueprint('verbs', __name__)
logger = logging.getLogger(__name__)
def _open_stream(namespace, repository, image_list):
def get_next_layer():
for current_image_id in image_list:
current_image_entry = model.get_repo_image(namespace, repository, current_image_id)
current_image_path = store.image_layer_path(current_image_entry.storage.uuid)
current_image_stream = store.stream_read_file(current_image_entry.storage.locations,
current_image_path)
logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
yield current_image_stream
stream = GzipWrap(StreamLayerMerger(get_next_layer).get_generator())
return stream.read
def _write_image_to_storage(namespace, repository, locations, image_id, queue_file):
image_path = store.image_layer_path(image_id)
store.stream_write(locations, image_path, queue_file)
queue_file.close()
@verbs.route('/<namespace>/<repository>/<tag>/squash', methods=['GET'])
@process_auth
@ -42,22 +62,43 @@ def get_squashed_tag(namespace, repository, tag, headers):
unhashed = str(repo_image.storage.id) + ':' + app.config['SECRET_KEY']
synthetic_image_id = hashlib.sha256(unhashed).hexdigest()
# Check to see if the synthetic image ID exists in storage. If so, we just return a 302.
logger.debug('Looking up synthetic image %s', synthetic_image_id)
locations = repo_image.storage.locations
saved_image_path = store.image_layer_path(synthetic_image_id)
if store.exists(locations, saved_image_path):
logger.debug('Synthetic image %s exists in storage', synthetic_image_id)
download_url = store.get_direct_download_url(locations, saved_image_path)
if download_url:
logger.debug('Redirecting to download URL for synthetic image %s', synthetic_image_id)
return redirect(download_url, code=302)
logger.debug('Sending cached synthetic image %s', synthetic_image_id)
return send_file(store.stream_read_file(locations, saved_image_path))
# Load the ancestry for the image.
logger.debug('Building and returning synthetic image %s', synthetic_image_id)
uuid = repo_image.storage.uuid
ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
full_image_list = json.loads(ancestry_data)
def get_next_layer():
for current_image_id in full_image_list:
current_image_entry = model.get_repo_image(namespace, repository, current_image_id)
current_image_path = store.image_layer_path(current_image_entry.storage.uuid)
current_image_stream = store.stream_read_file(current_image_entry.storage.locations,
current_image_path)
logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
yield current_image_stream
# Create a queue process to generate the data. The queue files will read from the process
# and send the results to the client and storage.
args = (namespace, repository, full_image_list)
queue_process = QueueProcess(_open_stream, 8 * 1024, 10 * 1024 * 1024, args) # 8K/10M chunk/max
stream = GzipWrap(StreamLayerMerger(get_next_layer).get_generator())
return send_file(stream)
client_queue_file = QueueFile(queue_process.create_queue(), 'client')
storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')
# Start building.
queue_process.run()
# Start the storage saving.
storage_args = (namespace, repository, locations, synthetic_image_id, storage_queue_file)
QueueProcess.run_process(_write_image_to_storage, storage_args)
# Return the client's data.
return send_file(client_queue_file)
abort(403)