quay/endpoints/verbs.py

import logging
import json
import hashlib

from flask import redirect, Blueprint, abort, send_file

from app import storage as store, app
from auth.auth import process_auth
from auth.permissions import ReadRepositoryPermission
from data import model
from data import database

from util.queuefile import QueueFile
from util.queueprocess import QueueProcess
from util.gzipwrap import GzipWrap
from util.dockerloadformat import build_docker_load_stream


verbs = Blueprint('verbs', __name__)
logger = logging.getLogger(__name__)


def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_list):
  def get_next_image():
    for current_image_id in image_list:
      yield model.get_repo_image(namespace, repository, current_image_id)

  def get_next_layer():
    for current_image_id in image_list:
      current_image_entry = model.get_repo_image(namespace, repository, current_image_id)
      current_image_path = store.image_layer_path(current_image_entry.storage.uuid)
      current_image_stream = store.stream_read_file(current_image_entry.storage.locations,
                                                    current_image_path)
        
      logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
      yield current_image_stream

  database.configure(app.config)
  stream = build_docker_load_stream(namespace, repository, tag, synthetic_image_id, image_json,
    get_next_image, get_next_layer)

  return stream.read


def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, queue_file):
  database.configure(app.config)

  def handle_exception(ex):
    model.delete_derived_storage_by_uuid(linked_storage_uuid)

  queue_file.add_exception_handler(handle_exception)

  image_path = store.image_layer_path(linked_storage_uuid)
  store.stream_write(linked_locations, image_path, queue_file)
  queue_file.close()

  if not queue_file.raised_exception:
    done_uploading = model.get_storage_by_uuid(linked_storage_uuid)
    done_uploading.uploading = False
    done_uploading.save()


@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
@process_auth
def get_squashed_tag(namespace, repository, tag):
  permission = ReadRepositoryPermission(namespace, repository)
  if permission.can() or model.repository_is_public(namespace, repository):
    # Lookup the requested tag.
    tag_image = model.get_tag_image(namespace, repository, tag)
    if not tag_image:
      abort(404)

    # Lookup the tag's image and storage.
    repo_image = model.get_repo_image(namespace, repository, tag_image.docker_image_id)
    if not repo_image:
      abort(404)

    derived = model.find_or_create_derived_storage(repo_image.storage, 'squash',
                                                   store.preferred_locations[0])
    if not derived.uploading:
      logger.debug('Derived image %s exists in storage', derived.uuid)
      derived_layer_path = store.image_layer_path(derived.uuid)
      download_url = store.get_direct_download_url(derived.locations, derived_layer_path)
      if download_url:
        logger.debug('Redirecting to download URL for derived image %s', derived.uuid)
        return redirect(download_url)

      logger.debug('Sending cached derived image %s', derived.uuid)
      return send_file(store.stream_read_file(derived.locations, derived_layer_path))

    # Load the ancestry for the image.
    logger.debug('Building and returning derived image %s', derived.uuid)
    uuid = repo_image.storage.uuid
    ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
    full_image_list = json.loads(ancestry_data)

    # Load the image's JSON layer.
    image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
    image_json = json.loads(image_json_data)

    # Calculate a synthetic image ID.
    synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':squash').hexdigest()

    # Create a queue process to generate the data. The queue files will read from the process
    # and send the results to the client and storage.
    args = (namespace, repository, tag, synthetic_image_id, image_json, full_image_list)
    queue_process = QueueProcess(_open_stream, 8 * 1024, 10 * 1024 * 1024, args) # 8K/10M chunk/max

    client_queue_file = QueueFile(queue_process.create_queue(), 'client')
    storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')

    # Start building.
    queue_process.run()

    # Start the storage saving.
    storage_args = (derived.uuid, derived.locations, storage_queue_file)
    QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args)

    # Return the client's data.
    return send_file(client_queue_file)

  abort(403)