Work in progress: Docker -> ACI conversion
This commit is contained in:
Joseph Schorr
parent
df9a417207
commit
6ed28930b2
13 changed files with 424 additions and 162 deletions
|
|
@ -16,12 +16,15 @@ from storage import Storage
|
|||
from util.queuefile import QueueFile
|
||||
from util.queueprocess import QueueProcess
|
||||
from util.gzipwrap import GzipWrap
|
||||
from util.dockerloadformat import build_docker_load_stream
|
||||
from formats.squashed import SquashedDockerImage
|
||||
from formats.aci import ACIImage
|
||||
|
||||
|
||||
verbs = Blueprint('verbs', __name__)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, image_id_list):
|
||||
def _open_stream(formatter, namespace, repository, tag, synthetic_image_id, image_json,
|
||||
image_id_list):
|
||||
store = Storage(app)
|
||||
|
||||
# For performance reasons, we load the full image list here, cache it, then disconnect from
|
||||
|
|
@ -45,17 +48,17 @@ def _open_stream(namespace, repository, tag, synthetic_image_id, image_json, ima
|
|||
logger.debug('Returning image layer %s: %s' % (current_image_id, current_image_path))
|
||||
yield current_image_stream
|
||||
|
||||
stream = build_docker_load_stream(namespace, repository, tag, synthetic_image_id, image_json,
|
||||
stream = formatter.build_stream(namespace, repository, tag, synthetic_image_id, image_json,
|
||||
get_next_image, get_next_layer)
|
||||
|
||||
return stream.read
|
||||
|
||||
|
||||
def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, queue_file):
|
||||
def _write_synthetic_image_to_storage(verb, linked_storage_uuid, linked_locations, queue_file):
|
||||
store = Storage(app)
|
||||
|
||||
def handle_exception(ex):
|
||||
logger.debug('Exception when building squashed image %s: %s', linked_storage_uuid, ex)
|
||||
logger.debug('Exception when building %s image %s: %s', verb, linked_storage_uuid, ex)
|
||||
|
||||
with database.UseThenDisconnect(app.config):
|
||||
model.delete_derived_storage_by_uuid(linked_storage_uuid)
|
||||
|
|
@ -73,11 +76,13 @@ def _write_synthetic_image_to_storage(linked_storage_uuid, linked_locations, que
|
|||
done_uploading.save()
|
||||
|
||||
|
||||
@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
|
||||
@process_auth
|
||||
def get_squashed_tag(namespace, repository, tag):
|
||||
def _repo_verb(namespace, repository, tag, verb, formatter, checker=None, **kwargs):
|
||||
permission = ReadRepositoryPermission(namespace, repository)
|
||||
if permission.can() or model.repository_is_public(namespace, repository):
|
||||
|
||||
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
# TODO: renable auth!
|
||||
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
if True or permission.can() or model.repository_is_public(namespace, repository):
|
||||
# Lookup the requested tag.
|
||||
try:
|
||||
tag_image = model.get_tag_image(namespace, repository, tag)
|
||||
|
|
@ -89,38 +94,54 @@ def get_squashed_tag(namespace, repository, tag):
|
|||
if not repo_image:
|
||||
abort(404)
|
||||
|
||||
# Log the action.
|
||||
track_and_log('repo_verb', repo_image.repository, tag=tag, verb='squash')
|
||||
|
||||
# If there is a data checker, call it first.
|
||||
store = Storage(app)
|
||||
derived = model.find_or_create_derived_storage(repo_image.storage, 'squash',
|
||||
uuid = repo_image.storage.uuid
|
||||
image_json = None
|
||||
|
||||
if checker is not None:
|
||||
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
image_json = json.loads(image_json_data)
|
||||
|
||||
if not checker(image_json):
|
||||
logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb)
|
||||
abort(404)
|
||||
|
||||
# Log the action.
|
||||
track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs)
|
||||
|
||||
derived = model.find_or_create_derived_storage(repo_image.storage, verb,
|
||||
store.preferred_locations[0])
|
||||
if not derived.uploading:
|
||||
logger.debug('Derived image %s exists in storage', derived.uuid)
|
||||
|
||||
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
# TODO: renable caching!
|
||||
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
if False and not derived.uploading:
|
||||
logger.debug('Derived %s image %s exists in storage', verb, derived.uuid)
|
||||
derived_layer_path = store.image_layer_path(derived.uuid)
|
||||
download_url = store.get_direct_download_url(derived.locations, derived_layer_path)
|
||||
if download_url:
|
||||
logger.debug('Redirecting to download URL for derived image %s', derived.uuid)
|
||||
logger.debug('Redirecting to download URL for derived %s image %s', verb, derived.uuid)
|
||||
return redirect(download_url)
|
||||
|
||||
# Close the database handle here for this process before we send the long download.
|
||||
database.close_db_filter(None)
|
||||
|
||||
logger.debug('Sending cached derived image %s', derived.uuid)
|
||||
logger.debug('Sending cached derived %s image %s', verb, derived.uuid)
|
||||
return send_file(store.stream_read_file(derived.locations, derived_layer_path))
|
||||
|
||||
# Load the ancestry for the image.
|
||||
logger.debug('Building and returning derived image %s', derived.uuid)
|
||||
uuid = repo_image.storage.uuid
|
||||
logger.debug('Building and returning derived %s image %s', verb, derived.uuid)
|
||||
ancestry_data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
|
||||
full_image_list = json.loads(ancestry_data)
|
||||
|
||||
# Load the image's JSON layer.
|
||||
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
image_json = json.loads(image_json_data)
|
||||
if not image_json:
|
||||
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
image_json = json.loads(image_json_data)
|
||||
|
||||
# Calculate a synthetic image ID.
|
||||
synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':squash').hexdigest()
|
||||
synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest()
|
||||
|
||||
# Create a queue process to generate the data. The queue files will read from the process
|
||||
# and send the results to the client and storage.
|
||||
|
|
@ -128,7 +149,7 @@ def get_squashed_tag(namespace, repository, tag):
|
|||
# Close any existing DB connection once the process has exited.
|
||||
database.close_db_filter(None)
|
||||
|
||||
args = (namespace, repository, tag, synthetic_image_id, image_json, full_image_list)
|
||||
args = (formatter, namespace, repository, tag, synthetic_image_id, image_json, full_image_list)
|
||||
queue_process = QueueProcess(_open_stream,
|
||||
8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max
|
||||
args, finished=_cleanup)
|
||||
|
|
@ -140,7 +161,7 @@ def get_squashed_tag(namespace, repository, tag):
|
|||
queue_process.run()
|
||||
|
||||
# Start the storage saving.
|
||||
storage_args = (derived.uuid, derived.locations, storage_queue_file)
|
||||
storage_args = (verb, derived.uuid, derived.locations, storage_queue_file)
|
||||
QueueProcess.run_process(_write_synthetic_image_to_storage, storage_args, finished=_cleanup)
|
||||
|
||||
# Close the database handle here for this process before we send the long download.
|
||||
|
|
@ -150,3 +171,29 @@ def get_squashed_tag(namespace, repository, tag):
|
|||
return send_file(client_queue_file)
|
||||
|
||||
abort(403)
|
||||
|
||||
|
||||
@verbs.route('/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/', methods=['GET'])
|
||||
@process_auth
|
||||
def get_rocket_image(server, namespace, repository, tag, os, arch):
|
||||
def checker(image_json):
|
||||
# Verify the architecture and os.
|
||||
operating_system = image_json.get('os', 'linux')
|
||||
if operating_system != os:
|
||||
return False
|
||||
|
||||
architecture = image_json.get('architecture', 'amd64')
|
||||
if architecture != arch:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
return _repo_verb(namespace, repository, tag, 'aci', ACIImage(),
|
||||
checker=checker, os=os, arch=arch)
|
||||
|
||||
|
||||
@verbs.route('/squash/<namespace>/<repository>/<tag>', methods=['GET'])
|
||||
@process_auth
|
||||
def get_squashed_tag(namespace, repository, tag):
|
||||
return _repo_verb(namespace, repository, tag, 'squash', SquashedDockerImage())
|
||||
|
||||
|
|
|
|||
Reference in a new issue