Add the basics of geographic data distribution and get the tests to work.
This commit is contained in:
Jake Moshenko
parent
2bf12996f5
commit
bf98575feb
23 changed files with 198 additions and 100 deletions
|
|
@ -106,14 +106,14 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
path = store.image_layer_path(repo_image.storage.uuid)
|
||||
|
||||
profile.debug('Looking up the direct download URL')
|
||||
direct_download_url = store.get_direct_download_url(path)
|
||||
direct_download_url = store.get_direct_download_url(repo_image.storage.locations, path)
|
||||
|
||||
if direct_download_url:
|
||||
profile.debug('Returning direct download URL')
|
||||
return redirect(direct_download_url)
|
||||
|
||||
profile.debug('Streaming layer data')
|
||||
return Response(store.stream_read(path), headers=headers)
|
||||
return Response(store.stream_read(repo_image.storage.locations, path), headers=headers)
|
||||
except (IOError, AttributeError):
|
||||
profile.debug('Image not found')
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
|
|
@ -136,7 +136,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
try:
|
||||
profile.debug('Retrieving image data')
|
||||
uuid = repo_image.storage.uuid
|
||||
json_data = store.get_content(store.image_json_path(uuid))
|
||||
json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
except (IOError, AttributeError):
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
|
@ -144,7 +144,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
profile.debug('Retrieving image path info')
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
|
||||
if (store.exists(layer_path) and not
|
||||
if (store.exists(repo_image.storage.locations, layer_path) and not
|
||||
image_is_uploading(repo_image)):
|
||||
exact_abort(409, 'Image already exists')
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
sr.add_handler(store_hndlr)
|
||||
h, sum_hndlr = checksums.simple_checksum_handler(json_data)
|
||||
sr.add_handler(sum_hndlr)
|
||||
store.stream_write(layer_path, sr)
|
||||
store.stream_write(repo_image.storage.locations, layer_path, sr)
|
||||
csums.append('sha256:{0}'.format(h.hexdigest()))
|
||||
|
||||
try:
|
||||
|
|
@ -231,7 +231,7 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
uuid = repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up repo layer data')
|
||||
if not store.exists(store.image_json_path(uuid)):
|
||||
if not store.exists(repo_image.storage.locations, store.image_json_path(uuid)):
|
||||
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
|
||||
|
||||
profile.debug('Marking image path')
|
||||
|
|
@ -283,7 +283,8 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
|
||||
profile.debug('Looking up repo layer data')
|
||||
try:
|
||||
data = store.get_content(store.image_json_path(repo_image.storage.uuid))
|
||||
uuid = repo_image.storage.uuid
|
||||
data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
|
||||
except (IOError, AttributeError):
|
||||
flask_abort(404)
|
||||
|
||||
|
|
@ -313,7 +314,8 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
|
||||
profile.debug('Looking up image data')
|
||||
try:
|
||||
data = store.get_content(store.image_ancestry_path(repo_image.storage.uuid))
|
||||
uuid = repo_image.storage.uuid
|
||||
data = store.get_content(repo_image.storage.locations, store.image_ancestry_path(uuid))
|
||||
except (IOError, AttributeError):
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
|
@ -326,17 +328,15 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
return response
|
||||
|
||||
|
||||
def generate_ancestry(image_id, uuid, parent_id=None,
|
||||
parent_uuid=None):
|
||||
def generate_ancestry(image_id, uuid, locations, parent_id=None, parent_uuid=None,
|
||||
parent_locations=None):
|
||||
if not parent_id:
|
||||
store.put_content(store.image_ancestry_path(uuid),
|
||||
json.dumps([image_id]))
|
||||
store.put_content(locations, store.image_ancestry_path(uuid), json.dumps([image_id]))
|
||||
return
|
||||
data = store.get_content(store.image_ancestry_path(parent_uuid))
|
||||
data = store.get_content(parent_locations, store.image_ancestry_path(parent_uuid))
|
||||
data = json.loads(data)
|
||||
data.insert(0, image_id)
|
||||
store.put_content(store.image_ancestry_path(uuid),
|
||||
json.dumps(data))
|
||||
store.put_content(locations, store.image_ancestry_path(uuid), json.dumps(data))
|
||||
|
||||
|
||||
def store_checksum(image_storage, checksum):
|
||||
|
|
@ -393,7 +393,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
profile.debug('Looking up parent image data')
|
||||
|
||||
if (parent_id and not
|
||||
store.exists(store.image_json_path(parent_uuid))):
|
||||
store.exists(parent_image.storage.locations, store.image_json_path(parent_uuid))):
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
|
||||
|
|
@ -401,7 +401,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
json_path = store.image_json_path(uuid)
|
||||
|
||||
profile.debug('Checking if image already exists')
|
||||
if (store.exists(json_path) and not
|
||||
if (store.exists(repo_image.storage.locations, json_path) and not
|
||||
image_is_uploading(repo_image)):
|
||||
exact_abort(409, 'Image already exists')
|
||||
|
||||
|
|
@ -424,10 +424,11 @@ def put_image_json(namespace, repository, image_id):
|
|||
parent_image)
|
||||
|
||||
profile.debug('Putting json path')
|
||||
store.put_content(json_path, request.data)
|
||||
store.put_content(repo_image.storage.locations, json_path, request.data)
|
||||
|
||||
profile.debug('Generating image ancestry')
|
||||
generate_ancestry(image_id, uuid, parent_id, parent_uuid)
|
||||
generate_ancestry(image_id, uuid, repo_image.storage.locations, parent_id, parent_uuid,
|
||||
parent_image.storage.locations)
|
||||
|
||||
profile.debug('Done')
|
||||
return make_response('true', 200)
|
||||
|
|
@ -442,7 +443,7 @@ def process_image_changes(namespace, repository, image_id):
|
|||
image_diffs_path = store.image_file_diffs_path(uuid)
|
||||
image_trie_path = store.image_file_trie_path(uuid)
|
||||
|
||||
if store.exists(image_diffs_path):
|
||||
if store.exists(repo_image.storage.locations, image_diffs_path):
|
||||
logger.debug('Diffs already exist for image: %s' % image_id)
|
||||
return image_trie_path
|
||||
|
||||
|
|
@ -452,18 +453,18 @@ def process_image_changes(namespace, repository, image_id):
|
|||
# Compute the diffs and fs for the parent first if necessary
|
||||
parent_trie_path = None
|
||||
if parents:
|
||||
parent_trie_path = process_image_changes(namespace, repository,
|
||||
parents[-1].docker_image_id)
|
||||
parent_trie_path, parent_locations = process_image_changes(namespace, repository,
|
||||
parents[-1].docker_image_id)
|
||||
|
||||
# Read in the collapsed layer state of the filesystem for the parent
|
||||
parent_trie = changes.empty_fs()
|
||||
if parent_trie_path:
|
||||
parent_trie_bytes = store.get_content(parent_trie_path)
|
||||
parent_trie_bytes = store.get_content(parent_locations, parent_trie_path)
|
||||
parent_trie.frombytes(parent_trie_bytes)
|
||||
|
||||
# Read in the file entries from the layer tar file
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
with store.stream_read_file(layer_path) as layer_tar_stream:
|
||||
with store.stream_read_file(image.storage.locations, layer_path) as layer_tar_stream:
|
||||
removed_files = set()
|
||||
layer_files = changes.files_and_dirs_from_tar(layer_tar_stream,
|
||||
removed_files)
|
||||
|
|
@ -473,7 +474,7 @@ def process_image_changes(namespace, repository, image_id):
|
|||
(new_trie, added, changed, removed) = new_metadata
|
||||
|
||||
# Write out the new trie
|
||||
store.put_content(image_trie_path, new_trie.tobytes())
|
||||
store.put_content(image.storage.locations, image_trie_path, new_trie.tobytes())
|
||||
|
||||
# Write out the diffs
|
||||
diffs = {}
|
||||
|
|
@ -481,6 +482,6 @@ def process_image_changes(namespace, repository, image_id):
|
|||
for section, source_trie in zip(sections, new_metadata[1:]):
|
||||
diffs[section] = list(source_trie)
|
||||
diffs[section].sort()
|
||||
store.put_content(image_diffs_path, json.dumps(diffs, indent=2))
|
||||
store.put_content(image.storage.locations, image_diffs_path, json.dumps(diffs, indent=2))
|
||||
|
||||
return image_trie_path
|
||||
return image_trie_path, image.storage.locations
|
||||
|
|
|
|||
Reference in a new issue