Get the base image stuff working. Checkpoint before fixing the tests.
This commit is contained in:
jakedt
parent
5742e6ea4e
commit
b619356907
120 changed files with 305 additions and 261 deletions
|
|
@ -44,8 +44,12 @@ def require_completion(f):
|
|||
"""This make sure that the image push correctly finished."""
|
||||
@wraps(f)
|
||||
def wrapper(namespace, repository, *args, **kwargs):
|
||||
if store.exists(store.image_mark_path(namespace, repository,
|
||||
kwargs['image_id'])):
|
||||
image_id = kwargs['image_id']
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
if store.exists(store.image_mark_path(namespace, repository, image_id,
|
||||
uuid)):
|
||||
abort(400, 'Image %(image_id)s is being uploaded, retry later',
|
||||
issue='upload-in-progress', image_id=kwargs['image_id'])
|
||||
|
||||
|
|
@ -85,14 +89,18 @@ def set_cache_headers(f):
|
|||
def get_image_layer(namespace, repository, image_id, headers):
|
||||
permission = ReadRepositoryPermission(namespace, repository)
|
||||
if permission.can() or model.repository_is_public(namespace, repository):
|
||||
path = store.image_layer_path(namespace, repository, image_id)
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
direct_download_url = store.get_direct_download_url(path)
|
||||
if direct_download_url:
|
||||
return redirect(direct_download_url)
|
||||
try:
|
||||
return Response(store.stream_read(path), headers=headers)
|
||||
except IOError:
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
abort(403)
|
||||
|
||||
|
|
@ -105,14 +113,17 @@ def put_image_layer(namespace, repository, image_id):
|
|||
if not permission.can():
|
||||
abort(403)
|
||||
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
try:
|
||||
json_data = store.get_content(store.image_json_path(namespace, repository,
|
||||
image_id))
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
layer_path = store.image_layer_path(namespace, repository, image_id)
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id)
|
||||
layer_path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
|
||||
if store.exists(layer_path) and not store.exists(mark_path):
|
||||
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
|
||||
|
|
@ -149,7 +160,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
try:
|
||||
checksum = store.get_content(store.image_checksum_path(namespace,
|
||||
repository,
|
||||
image_id))
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
# We don't have a checksum stored yet, that's fine skipping the check.
|
||||
# Not removing the mark though, image is not downloadable yet.
|
||||
|
|
@ -193,15 +204,18 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
abort(400, 'Checksum not found in Cookie for image %(imaage_id)s',
|
||||
issue='missing-checksum-cookie', image_id=image_id)
|
||||
|
||||
if not store.exists(store.image_json_path(namespace, repository, image_id)):
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
if not store.exists(store.image_json_path(namespace, repository, image_id,
|
||||
uuid)):
|
||||
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
|
||||
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id)
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
if not store.exists(mark_path):
|
||||
abort(409, 'Cannot set checksum for image %(image_id)s',
|
||||
issue='image-write-error', image_id=image_id)
|
||||
|
||||
err = store_checksum(namespace, repository, image_id, checksum)
|
||||
err = store_checksum(namespace, repository, image_id, uuid, checksum)
|
||||
if err:
|
||||
abort(400, err)
|
||||
|
||||
|
|
@ -238,20 +252,24 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
repository):
|
||||
abort(403)
|
||||
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
try:
|
||||
data = store.get_content(store.image_json_path(namespace, repository,
|
||||
image_id))
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
flask_abort(404)
|
||||
|
||||
try:
|
||||
size = store.get_size(store.image_layer_path(namespace, repository,
|
||||
image_id))
|
||||
image_id, uuid))
|
||||
headers['X-Docker-Size'] = str(size)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
checksum_path = store.image_checksum_path(namespace, repository, image_id)
|
||||
checksum_path = store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid)
|
||||
if store.exists(checksum_path):
|
||||
headers['X-Docker-Checksum'] = store.get_content(checksum_path)
|
||||
|
||||
|
|
@ -271,39 +289,45 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
repository):
|
||||
abort(403)
|
||||
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
try:
|
||||
data = store.get_content(store.image_ancestry_path(namespace, repository,
|
||||
image_id))
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
response = make_response(json.dumps(json.loads(data)), 200)
|
||||
response.headers.extend(headers)
|
||||
return response
|
||||
|
||||
|
||||
def generate_ancestry(namespace, repository, image_id, parent_id=None):
|
||||
def generate_ancestry(namespace, repository, image_id, uuid, parent_id=None,
|
||||
parent_uuid=None):
|
||||
if not parent_id:
|
||||
store.put_content(store.image_ancestry_path(namespace, repository,
|
||||
image_id),
|
||||
image_id, uuid),
|
||||
json.dumps([image_id]))
|
||||
return
|
||||
data = store.get_content(store.image_ancestry_path(namespace, repository,
|
||||
parent_id))
|
||||
parent_id, parent_uuid))
|
||||
data = json.loads(data)
|
||||
data.insert(0, image_id)
|
||||
store.put_content(store.image_ancestry_path(namespace, repository,
|
||||
image_id),
|
||||
image_id, uuid),
|
||||
json.dumps(data))
|
||||
|
||||
|
||||
def store_checksum(namespace, repository, image_id, checksum):
|
||||
def store_checksum(namespace, repository, image_id, uuid, checksum):
|
||||
checksum_parts = checksum.split(':')
|
||||
if len(checksum_parts) != 2:
|
||||
return 'Invalid checksum format'
|
||||
|
||||
# We store the checksum
|
||||
checksum_path = store.image_checksum_path(namespace, repository, image_id)
|
||||
checksum_path = store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid)
|
||||
store.put_content(checksum_path, checksum)
|
||||
|
||||
|
||||
|
|
@ -327,58 +351,69 @@ def put_image_json(namespace, repository, image_id):
|
|||
abort(400, 'Missing key `id` in JSON for image: %(image_id)s',
|
||||
issue='invalid-request', image_id=image_id)
|
||||
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
# Read the checksum
|
||||
checksum = request.headers.get('X-Docker-Checksum')
|
||||
if checksum:
|
||||
# Storing the checksum is optional at this stage
|
||||
err = store_checksum(namespace, repository, image_id, checksum)
|
||||
err = store_checksum(namespace, repository, image_id, uuid, checksum)
|
||||
if err:
|
||||
abort(400, err, issue='write-error')
|
||||
|
||||
else:
|
||||
# We cleanup any old checksum in case it's a retry after a fail
|
||||
store.remove(store.image_checksum_path(namespace, repository, image_id))
|
||||
store.remove(store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid))
|
||||
if image_id != data['id']:
|
||||
abort(400, 'JSON data contains invalid id for image: %(image_id)s',
|
||||
issue='invalid-request', image_id=image_id)
|
||||
|
||||
parent_id = data.get('parent')
|
||||
|
||||
parent_image = None
|
||||
if parent_id:
|
||||
parent_image = model.get_repo_image(namespace, repository, parent_id)
|
||||
parent_uuid = (parent_image and parent_image.storage and
|
||||
parent_image.storage.uuid)
|
||||
|
||||
if (parent_id and not
|
||||
store.exists(store.image_json_path(namespace, repository, parent_id))):
|
||||
store.exists(store.image_json_path(namespace, repository, parent_id,
|
||||
parent_uuid))):
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
|
||||
json_path = store.image_json_path(namespace, repository, image_id)
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id)
|
||||
json_path = store.image_json_path(namespace, repository, image_id, uuid)
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
if store.exists(json_path) and not store.exists(mark_path):
|
||||
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
|
||||
|
||||
# If we reach that point, it means that this is a new image or a retry
|
||||
# on a failed push
|
||||
# save the metadata
|
||||
if parent_id:
|
||||
parent_obj = model.get_image_by_id(namespace, repository, parent_id)
|
||||
else:
|
||||
parent_obj = None
|
||||
|
||||
command_list = data.get('container_config', {}).get('Cmd', None)
|
||||
command = json.dumps(command_list) if command_list else None
|
||||
model.set_image_metadata(image_id, namespace, repository,
|
||||
data.get('created'), data.get('comment'), command,
|
||||
parent_obj)
|
||||
parent_image)
|
||||
store.put_content(mark_path, 'true')
|
||||
store.put_content(json_path, request.data)
|
||||
generate_ancestry(namespace, repository, image_id, parent_id)
|
||||
generate_ancestry(namespace, repository, image_id, uuid, parent_id,
|
||||
parent_uuid)
|
||||
return make_response('true', 200)
|
||||
|
||||
|
||||
def process_image_changes(namespace, repository, image_id):
|
||||
logger.debug('Generating diffs for image: %s' % image_id)
|
||||
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
image_diffs_path = store.image_file_diffs_path(namespace, repository,
|
||||
image_id)
|
||||
image_id, uuid)
|
||||
image_trie_path = store.image_file_trie_path(namespace, repository,
|
||||
image_id)
|
||||
image_id, uuid)
|
||||
|
||||
if store.exists(image_diffs_path):
|
||||
logger.debug('Diffs already exist for image: %s' % image_id)
|
||||
|
|
@ -400,7 +435,7 @@ def process_image_changes(namespace, repository, image_id):
|
|||
parent_trie.frombytes(parent_trie_bytes)
|
||||
|
||||
# Read in the file entries from the layer tar file
|
||||
layer_path = store.image_layer_path(namespace, repository, image_id)
|
||||
layer_path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
with store.stream_read_file(layer_path) as layer_tar_stream:
|
||||
removed_files = set()
|
||||
layer_files = changes.files_and_dirs_from_tar(layer_tar_stream,
|
||||
|
|
|
|||
Reference in a new issue