Switch the checksums to use the registry computed value, remove all assumptions of namespaced paths for legacy storage, fix an upload race condition in the registry code.
This commit is contained in:
Jake Moshenko
parent
246a216f80
commit
78c5aec5b9
11 changed files with 112 additions and 264 deletions
|
|
@ -228,12 +228,7 @@ class Image(BaseModel):
|
|||
# security reasons. So rather than Repository <-> Image being many to many
|
||||
# each image now belongs to exactly one repository.
|
||||
docker_image_id = CharField()
|
||||
checksum = CharField(null=True)
|
||||
created = DateTimeField(null=True)
|
||||
comment = TextField(null=True)
|
||||
command = TextField(null=True)
|
||||
repository = ForeignKeyField(Repository)
|
||||
image_size = BigIntegerField(null=True)
|
||||
|
||||
# '/' separated list of ancestory ids, e.g. /1/2/6/7/10/
|
||||
ancestors = CharField(index=True, default='/', max_length=64535, null=True)
|
||||
|
|
@ -244,7 +239,7 @@ class Image(BaseModel):
|
|||
database = db
|
||||
indexes = (
|
||||
# we don't really want duplicates
|
||||
(('repository', 'docker_image_id'), False),
|
||||
(('repository', 'docker_image_id'), True),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1011,14 +1011,6 @@ def find_create_or_link_image(docker_image_id, repository, username,
|
|||
return new_image
|
||||
|
||||
|
||||
def set_image_checksum(docker_image_id, repository, checksum):
|
||||
fetched = Image.get(Image.docker_image_id == docker_image_id,
|
||||
Image.repository == repository)
|
||||
fetched.checksum = checksum
|
||||
fetched.save()
|
||||
return fetched
|
||||
|
||||
|
||||
def set_image_size(docker_image_id, namespace_name, repository_name,
|
||||
image_size):
|
||||
try:
|
||||
|
|
@ -1122,15 +1114,9 @@ def garbage_collect_repository(namespace_name, repository_name):
|
|||
for image_id_to_remove in to_remove:
|
||||
image_to_remove = all_images[image_id_to_remove]
|
||||
|
||||
if image_to_remove.storage and image_to_remove.storage.id:
|
||||
logger.debug('Adding image storage to the gc list: %s',
|
||||
image_to_remove.storage.uuid)
|
||||
uuids_to_check_for_gc.add(image_to_remove.storage.uuid)
|
||||
else:
|
||||
image_path = config.store.image_path(namespace_name, repository_name,
|
||||
image_to_remove.docker_image_id, None)
|
||||
logger.debug('Deleting image storage: %s', image_path)
|
||||
config.store.remove(image_path)
|
||||
logger.debug('Adding image storage to the gc list: %s',
|
||||
image_to_remove.storage.uuid)
|
||||
uuids_to_check_for_gc.add(image_to_remove.storage.uuid)
|
||||
|
||||
image_to_remove.delete_instance()
|
||||
|
||||
|
|
@ -1145,8 +1131,7 @@ def garbage_collect_repository(namespace_name, repository_name):
|
|||
for storage in storage_to_remove:
|
||||
logger.debug('Garbage collecting image storage: %s', storage.uuid)
|
||||
storage.delete_instance()
|
||||
image_path = config.store.image_path(namespace_name, repository_name,
|
||||
image_to_remove.docker_image_id, storage.uuid)
|
||||
image_path = config.store.image_path(storage.uuid)
|
||||
config.store.remove(image_path)
|
||||
|
||||
return len(to_remove)
|
||||
|
|
|
|||
|
|
@ -79,8 +79,7 @@ class RepositoryImageChanges(RepositoryParamResource):
|
|||
if not image:
|
||||
raise NotFound()
|
||||
|
||||
uuid = image.storage and image.storage.uuid
|
||||
diffs_path = store.image_file_diffs_path(namespace, repository, image_id, uuid)
|
||||
diffs_path = store.image_file_diffs_path(image.storage.uuid)
|
||||
|
||||
try:
|
||||
response_json = json.loads(store.get_content(diffs_path))
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ def update_user(username):
|
|||
@generate_headers(role='write')
|
||||
def create_repository(namespace, repository):
|
||||
profile.debug('Parsing image descriptions')
|
||||
image_descriptions = json.loads(request.data)
|
||||
image_descriptions = json.loads(request.data.decode('utf8'))
|
||||
|
||||
profile.debug('Looking up repository')
|
||||
repo = model.get_repository(namespace, repository)
|
||||
|
|
@ -292,13 +292,11 @@ def update_images(namespace, repository):
|
|||
abort(404, message='Unknown repository', issue='unknown-repo')
|
||||
|
||||
profile.debug('Parsing image data')
|
||||
image_with_checksums = json.loads(request.data)
|
||||
image_with_checksums = json.loads(request.data.decode('utf8'))
|
||||
|
||||
updated_tags = {}
|
||||
for image in image_with_checksums:
|
||||
profile.debug('Setting checksum for image id: %s to %s', image['id'], image['checksum'])
|
||||
updated_tags[image['Tag']] = image['id']
|
||||
model.set_image_checksum(image['id'], repo, image['checksum'])
|
||||
|
||||
if get_authenticated_user():
|
||||
profile.debug('Publishing push event')
|
||||
|
|
@ -366,7 +364,7 @@ def get_repository_images(namespace, repository):
|
|||
for image in model.get_repository_images(namespace, repository):
|
||||
new_image_view = {
|
||||
'id': image.docker_image_id,
|
||||
'checksum': image.checksum,
|
||||
'checksum': image.storage.checksum,
|
||||
}
|
||||
all_images.append(new_image_view)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,27 +38,29 @@ class SocketReader(object):
|
|||
return buf
|
||||
|
||||
|
||||
def image_is_uploading(namespace, repository, image_id, repo_image):
|
||||
if repo_image and repo_image.storage and repo_image.storage.uploading is not None:
|
||||
def image_is_uploading(repo_image):
|
||||
if repo_image is None:
|
||||
return False
|
||||
|
||||
if repo_image.storage.uploading is not None:
|
||||
return repo_image.storage.uploading
|
||||
|
||||
logger.warning('Setting legacy upload flag')
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
logger.warning('Checking legacy upload flag')
|
||||
mark_path = store.image_mark_path(repo_image.storage.uuid)
|
||||
return store.exists(mark_path)
|
||||
|
||||
|
||||
def mark_upload_complete(namespace, repository, image_id, repo_image):
|
||||
if repo_image and repo_image.storage and repo_image.storage.uploading is not None:
|
||||
repo_image.storage.uploading = False
|
||||
repo_image.storage.save()
|
||||
else:
|
||||
def set_uploading_flag(repo_image, is_image_uploading):
|
||||
if repo_image.storage.uploading is None and not is_image_uploading:
|
||||
logger.warning('Removing legacy upload flag')
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
mark_path = store.image_mark_path(namespace, repository, image_id, uuid)
|
||||
uuid = repo_image.storage.uuid
|
||||
mark_path = store.image_mark_path(uuid)
|
||||
if store.exists(mark_path):
|
||||
store.remove(mark_path)
|
||||
|
||||
repo_image.storage.uploading = is_image_uploading
|
||||
repo_image.storage.save()
|
||||
|
||||
|
||||
def require_completion(f):
|
||||
"""This make sure that the image push correctly finished."""
|
||||
|
|
@ -66,7 +68,7 @@ def require_completion(f):
|
|||
def wrapper(namespace, repository, *args, **kwargs):
|
||||
image_id = kwargs['image_id']
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
if image_is_uploading(namespace, repository, image_id, repo_image):
|
||||
if image_is_uploading(repo_image):
|
||||
abort(400, 'Image %(image_id)s is being uploaded, retry later',
|
||||
issue='upload-in-progress', image_id=kwargs['image_id'])
|
||||
|
||||
|
|
@ -111,21 +113,20 @@ def get_image_layer(namespace, repository, image_id, headers):
|
|||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up the layer path')
|
||||
path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
|
||||
profile.debug('Looking up the direct download URL')
|
||||
direct_download_url = store.get_direct_download_url(path)
|
||||
|
||||
if direct_download_url:
|
||||
profile.debug('Returning direct download URL')
|
||||
return redirect(direct_download_url)
|
||||
try:
|
||||
path = store.image_layer_path(repo_image.storage.uuid)
|
||||
|
||||
profile.debug('Looking up the direct download URL')
|
||||
direct_download_url = store.get_direct_download_url(path)
|
||||
|
||||
if direct_download_url:
|
||||
profile.debug('Returning direct download URL')
|
||||
return redirect(direct_download_url)
|
||||
|
||||
profile.debug('Streaming layer data')
|
||||
return Response(store.stream_read(path), headers=headers)
|
||||
except IOError:
|
||||
except (IOError, AttributeError):
|
||||
profile.debug('Image not found')
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
|
@ -144,21 +145,19 @@ def put_image_layer(namespace, repository, image_id):
|
|||
|
||||
profile.debug('Retrieving image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
try:
|
||||
profile.debug('Retrieving image data')
|
||||
json_data = store.get_content(store.image_json_path(namespace, repository,
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
uuid = repo_image.storage.uuid
|
||||
json_data = store.get_content(store.image_json_path(uuid))
|
||||
except (IOError, AttributeError):
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
profile.debug('Retrieving image path info')
|
||||
layer_path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
|
||||
if (store.exists(layer_path) and not
|
||||
image_is_uploading(namespace, repository, image_id, repo_image)):
|
||||
image_is_uploading(repo_image)):
|
||||
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
|
||||
|
||||
profile.debug('Storing layer data')
|
||||
|
|
@ -193,9 +192,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
'{0}'.format(e))
|
||||
|
||||
try:
|
||||
checksum = store.get_content(store.image_checksum_path(namespace,
|
||||
repository,
|
||||
image_id, uuid))
|
||||
checksum = store.get_content(store.image_checksum_path(uuid))
|
||||
except IOError:
|
||||
# We don't have a checksum stored yet, that's fine skipping the check.
|
||||
# Not removing the mark though, image is not downloadable yet.
|
||||
|
|
@ -209,7 +206,7 @@ def put_image_layer(namespace, repository, image_id):
|
|||
issue='checksum-mismatch', image_id=image_id)
|
||||
|
||||
# Checksum is ok, we remove the marker
|
||||
mark_upload_complete(namespace, repository, image_id, repo_image)
|
||||
set_uploading_flag(repo_image, False)
|
||||
|
||||
# The layer is ready for download, send a job to the work queue to
|
||||
# process it.
|
||||
|
|
@ -232,9 +229,11 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
if not permission.can():
|
||||
abort(403)
|
||||
|
||||
checksum = request.headers.get('X-Docker-Checksum')
|
||||
checksum = (request.headers.get('X-Docker-Checksum-Payload', None) or
|
||||
request.headers.get('X-Docker-Checksum'))
|
||||
if not checksum:
|
||||
abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum', image_id=image_id)
|
||||
abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum',
|
||||
image_id=image_id)
|
||||
|
||||
if not session.get('checksum'):
|
||||
abort(400, 'Checksum not found in Cookie for image %(image_id)s',
|
||||
|
|
@ -242,21 +241,19 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
uuid = repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up repo layer data')
|
||||
if not store.exists(store.image_json_path(namespace, repository, image_id,
|
||||
uuid)):
|
||||
if not store.exists(store.image_json_path(uuid)):
|
||||
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
|
||||
|
||||
profile.debug('Marking image path')
|
||||
if not image_is_uploading(namespace, repository, image_id, repo_image):
|
||||
if not image_is_uploading(repo_image):
|
||||
abort(409, 'Cannot set checksum for image %(image_id)s',
|
||||
issue='image-write-error', image_id=image_id)
|
||||
|
||||
profile.debug('Storing image checksum')
|
||||
err = store_checksum(namespace, repository, image_id, uuid, checksum)
|
||||
err = store_checksum(repo_image.storage, checksum)
|
||||
if err:
|
||||
abort(400, err)
|
||||
|
||||
|
|
@ -268,7 +265,7 @@ def put_image_checksum(namespace, repository, image_id):
|
|||
issue='checksum-mismatch', image_id=image_id)
|
||||
|
||||
# Checksum is ok, we remove the marker
|
||||
mark_upload_complete(namespace, repository, image_id, repo_image)
|
||||
set_uploading_flag(repo_image, False)
|
||||
|
||||
# The layer is ready for download, send a job to the work queue to
|
||||
# process it.
|
||||
|
|
@ -296,13 +293,11 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up repo layer data')
|
||||
try:
|
||||
data = store.get_content(store.image_json_path(namespace, repository,
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
data = store.get_content(store.image_json_path(repo_image.storage.uuid))
|
||||
except (IOError, AttributeError):
|
||||
flask_abort(404)
|
||||
|
||||
profile.debug('Looking up repo layer size')
|
||||
|
|
@ -312,12 +307,6 @@ def get_image_json(namespace, repository, image_id, headers):
|
|||
except OSError:
|
||||
pass
|
||||
|
||||
profile.debug('Retrieving checksum')
|
||||
checksum_path = store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid)
|
||||
if store.exists(checksum_path):
|
||||
headers['X-Docker-Checksum'] = store.get_content(checksum_path)
|
||||
|
||||
response = make_response(data, 200)
|
||||
response.headers.extend(headers)
|
||||
return response
|
||||
|
|
@ -337,13 +326,11 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
|
||||
profile.debug('Looking up image data')
|
||||
try:
|
||||
data = store.get_content(store.image_ancestry_path(namespace, repository,
|
||||
image_id, uuid))
|
||||
except IOError:
|
||||
data = store.get_content(store.image_ancestry_path(repo_image.storage.uuid))
|
||||
except (IOError, AttributeError):
|
||||
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
|
||||
image_id=image_id)
|
||||
|
||||
|
|
@ -355,32 +342,32 @@ def get_image_ancestry(namespace, repository, image_id, headers):
|
|||
return response
|
||||
|
||||
|
||||
def generate_ancestry(namespace, repository, image_id, uuid, parent_id=None,
|
||||
def generate_ancestry(image_id, uuid, parent_id=None,
|
||||
parent_uuid=None):
|
||||
if not parent_id:
|
||||
store.put_content(store.image_ancestry_path(namespace, repository,
|
||||
image_id, uuid),
|
||||
store.put_content(store.image_ancestry_path(uuid),
|
||||
json.dumps([image_id]))
|
||||
return
|
||||
data = store.get_content(store.image_ancestry_path(namespace, repository,
|
||||
parent_id, parent_uuid))
|
||||
data = store.get_content(store.image_ancestry_path(parent_uuid))
|
||||
data = json.loads(data)
|
||||
data.insert(0, image_id)
|
||||
store.put_content(store.image_ancestry_path(namespace, repository,
|
||||
image_id, uuid),
|
||||
store.put_content(store.image_ancestry_path(uuid),
|
||||
json.dumps(data))
|
||||
|
||||
|
||||
def store_checksum(namespace, repository, image_id, uuid, checksum):
|
||||
def store_checksum(image_storage, checksum):
|
||||
checksum_parts = checksum.split(':')
|
||||
if len(checksum_parts) != 2:
|
||||
return 'Invalid checksum format'
|
||||
|
||||
# We store the checksum
|
||||
checksum_path = store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid)
|
||||
checksum_path = store.image_checksum_path(image_storage.uuid)
|
||||
store.put_content(checksum_path, checksum)
|
||||
|
||||
# And store it in the db
|
||||
image_storage.checksum = checksum
|
||||
image_storage.save()
|
||||
|
||||
|
||||
@registry.route('/images/<image_id>/json', methods=['PUT'])
|
||||
@process_auth
|
||||
|
|
@ -393,9 +380,10 @@ def put_image_json(namespace, repository, image_id):
|
|||
|
||||
profile.debug('Parsing image JSON')
|
||||
try:
|
||||
data = json.loads(request.data)
|
||||
except json.JSONDecodeError:
|
||||
data = json.loads(request.data.decode('utf8'))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if not data or not isinstance(data, dict):
|
||||
abort(400, 'Invalid JSON for image: %(image_id)s\nJSON: %(json)s',
|
||||
issue='invalid-request', image_id=image_id, json=request.data)
|
||||
|
|
@ -406,22 +394,8 @@ def put_image_json(namespace, repository, image_id):
|
|||
|
||||
profile.debug('Looking up repo image')
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
uuid = repo_image.storage.uuid
|
||||
|
||||
# Read the checksum
|
||||
checksum = request.headers.get('X-Docker-Checksum')
|
||||
if checksum:
|
||||
# Storing the checksum is optional at this stage
|
||||
profile.debug('Storing image checksum')
|
||||
err = store_checksum(namespace, repository, image_id, uuid, checksum)
|
||||
if err:
|
||||
abort(400, err, issue='write-error')
|
||||
|
||||
else:
|
||||
# We cleanup any old checksum in case it's a retry after a fail
|
||||
profile.debug('Cleanup old checksum')
|
||||
store.remove(store.image_checksum_path(namespace, repository, image_id,
|
||||
uuid))
|
||||
if image_id != data['id']:
|
||||
abort(400, 'JSON data contains invalid id for image: %(image_id)s',
|
||||
issue='invalid-request', image_id=image_id)
|
||||
|
|
@ -433,26 +407,33 @@ def put_image_json(namespace, repository, image_id):
|
|||
profile.debug('Looking up parent image')
|
||||
parent_image = model.get_repo_image(namespace, repository, parent_id)
|
||||
|
||||
parent_uuid = (parent_image and parent_image.storage and
|
||||
parent_image.storage.uuid)
|
||||
parent_uuid = parent_image and parent_image.storage.uuid
|
||||
|
||||
if parent_id:
|
||||
profile.debug('Looking up parent image data')
|
||||
|
||||
if (parent_id and not
|
||||
store.exists(store.image_json_path(namespace, repository, parent_id,
|
||||
parent_uuid))):
|
||||
store.exists(store.image_json_path(parent_uuid))):
|
||||
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
|
||||
issue='invalid-request', image_id=image_id, parent_id=parent_id)
|
||||
|
||||
profile.debug('Looking up image storage paths')
|
||||
json_path = store.image_json_path(namespace, repository, image_id, uuid)
|
||||
json_path = store.image_json_path(uuid)
|
||||
|
||||
profile.debug('Checking if image already exists')
|
||||
if (store.exists(json_path) and not
|
||||
image_is_uploading(namespace, repository, image_id, repo_image)):
|
||||
image_is_uploading(repo_image)):
|
||||
abort(409, 'Image already exists', issue='image-exists', image_id=image_id)
|
||||
|
||||
set_uploading_flag(repo_image, True)
|
||||
|
||||
# We cleanup any old checksum in case it's a retry after a fail
|
||||
profile.debug('Cleanup old checksum')
|
||||
try:
|
||||
store.remove(store.image_checksum_path(uuid))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# If we reach that point, it means that this is a new image or a retry
|
||||
# on a failed push
|
||||
# save the metadata
|
||||
|
|
@ -468,8 +449,7 @@ def put_image_json(namespace, repository, image_id):
|
|||
store.put_content(json_path, request.data)
|
||||
|
||||
profile.debug('Generating image ancestry')
|
||||
generate_ancestry(namespace, repository, image_id, uuid, parent_id,
|
||||
parent_uuid)
|
||||
generate_ancestry(image_id, uuid, parent_id, parent_uuid)
|
||||
|
||||
profile.debug('Done')
|
||||
return make_response('true', 200)
|
||||
|
|
@ -479,12 +459,10 @@ def process_image_changes(namespace, repository, image_id):
|
|||
logger.debug('Generating diffs for image: %s' % image_id)
|
||||
|
||||
repo_image = model.get_repo_image(namespace, repository, image_id)
|
||||
uuid = repo_image and repo_image.storage and repo_image.storage.uuid
|
||||
uuid = repo_image.storage.uuid
|
||||
|
||||
image_diffs_path = store.image_file_diffs_path(namespace, repository,
|
||||
image_id, uuid)
|
||||
image_trie_path = store.image_file_trie_path(namespace, repository,
|
||||
image_id, uuid)
|
||||
image_diffs_path = store.image_file_diffs_path(uuid)
|
||||
image_trie_path = store.image_file_trie_path(uuid)
|
||||
|
||||
if store.exists(image_diffs_path):
|
||||
logger.debug('Diffs already exist for image: %s' % image_id)
|
||||
|
|
@ -506,7 +484,7 @@ def process_image_changes(namespace, repository, image_id):
|
|||
parent_trie.frombytes(parent_trie_bytes)
|
||||
|
||||
# Read in the file entries from the layer tar file
|
||||
layer_path = store.image_layer_path(namespace, repository, image_id, uuid)
|
||||
layer_path = store.image_layer_path(uuid)
|
||||
with store.stream_read_file(layer_path) as layer_tar_stream:
|
||||
removed_files = set()
|
||||
layer_files = changes.files_and_dirs_from_tar(layer_tar_stream,
|
||||
|
|
|
|||
|
|
@ -70,10 +70,10 @@ def __create_subtree(repo, structure, creator_username, parent):
|
|||
new_image = model.find_create_or_link_image(docker_image_id, repo, None,
|
||||
{})
|
||||
new_image.storage.uuid = IMAGE_UUIDS[image_num % len(IMAGE_UUIDS)]
|
||||
new_image.storage.uploading = False
|
||||
new_image.storage.checksum = checksum
|
||||
new_image.storage.save()
|
||||
|
||||
model.set_image_checksum(docker_image_id, repo, checksum)
|
||||
|
||||
creation_time = REFERENCE_DATE + timedelta(days=image_num)
|
||||
command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
|
||||
command = json.dumps(command_list) if command_list else None
|
||||
|
|
@ -85,9 +85,7 @@ def __create_subtree(repo, structure, creator_username, parent):
|
|||
random.randrange(1, 1024 * 1024 * 1024))
|
||||
|
||||
# Populate the diff file
|
||||
diff_path = store.image_file_diffs_path(repo.namespace, repo.name,
|
||||
docker_image_id,
|
||||
new_image.storage.uuid)
|
||||
diff_path = store.image_file_diffs_path(new_image.storage.uuid)
|
||||
source_diff = SAMPLE_DIFFS[image_num % len(SAMPLE_DIFFS)]
|
||||
|
||||
with open(source_diff, 'r') as source_file:
|
||||
|
|
|
|||
|
|
@ -29,41 +29,35 @@ class BaseStorage(object):
|
|||
|
||||
return tmpf, fn
|
||||
|
||||
def image_path(self, namespace, repository, image_id, storage_uuid):
|
||||
if storage_uuid:
|
||||
return '{0}/{1}/'.format(self.shared_images, storage_uuid)
|
||||
else:
|
||||
return '{0}/{1}/{2}/{3}/'.format(self.images, namespace, repository,
|
||||
image_id)
|
||||
def image_path(self, storage_uuid):
|
||||
return '{0}/{1}/'.format(self.shared_images, storage_uuid)
|
||||
|
||||
def image_json_path(self, namespace, repository, image_id, storage_uuid):
|
||||
base_path = self.image_path(namespace, repository, image_id, storage_uuid)
|
||||
def image_json_path(self, storage_uuid):
|
||||
base_path = self.image_path(storage_uuid)
|
||||
return '{0}json'.format(base_path)
|
||||
|
||||
def image_mark_path(self, namespace, repository, image_id, storage_uuid):
|
||||
base_path = self.image_path(namespace, repository, image_id, storage_uuid)
|
||||
def image_mark_path(self, storage_uuid):
|
||||
base_path = self.image_path(storage_uuid)
|
||||
return '{0}_inprogress'.format(base_path)
|
||||
|
||||
def image_checksum_path(self, namespace, repository, image_id, storage_uuid):
|
||||
base_path = self.image_path(namespace, repository, image_id, storage_uuid)
|
||||
def image_checksum_path(self, storage_uuid):
|
||||
base_path = self.image_path(storage_uuid)
|
||||
return '{0}_checksum'.format(base_path)
|
||||
|
||||
def image_layer_path(self, namespace, repository, image_id, storage_uuid):
|
||||
base_path = self.image_path(namespace, repository, image_id, storage_uuid)
|
||||
def image_layer_path(self, storage_uuid):
|
||||
base_path = self.image_path(storage_uuid)
|
||||
return '{0}layer'.format(base_path)
|
||||
|
||||
def image_ancestry_path(self, namespace, repository, image_id, storage_uuid):
|
||||
base_path = self.image_path(namespace, repository, image_id, storage_uuid)
|
||||
def image_ancestry_path(self, storage_uuid):
|
||||
base_path = self.image_path(storage_uuid)
|
||||
return '{0}ancestry'.format(base_path)
|
||||
|
||||
def image_file_trie_path(self, namespace, repository, image_id,
|
||||
storage_uuid):
|
||||
base_path = self.image_path(namespace, repository, image_id, storage_uuid)
|
||||
def image_file_trie_path(self, storage_uuid):
|
||||
base_path = self.image_path(storage_uuid)
|
||||
return '{0}files.trie'.format(base_path)
|
||||
|
||||
def image_file_diffs_path(self, namespace, repository, image_id,
|
||||
storage_uuid):
|
||||
base_path = self.image_path(namespace, repository, image_id, storage_uuid)
|
||||
def image_file_diffs_path(self, storage_uuid):
|
||||
base_path = self.image_path(storage_uuid)
|
||||
return '{0}diffs.json'.format(base_path)
|
||||
|
||||
def get_direct_download_url(self, path, expires_in=60):
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -113,11 +113,11 @@ class IndexTestSpec(object):
|
|||
def build_index_specs():
|
||||
return [
|
||||
IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID),
|
||||
PUBLIC_REPO, 200, 200, 200, 200),
|
||||
PUBLIC_REPO, 404, 404, 404, 404),
|
||||
IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID),
|
||||
PRIVATE_REPO),
|
||||
PRIVATE_REPO, 403, 403, 404, 404),
|
||||
IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID),
|
||||
ORG_REPO),
|
||||
ORG_REPO, 403, 403, 404, 404),
|
||||
|
||||
IndexTestSpec(url_for('registry.put_image_layer', image_id=FAKE_IMAGE_ID),
|
||||
PUBLIC_REPO, 403, 403, 403, 403).set_method('PUT'),
|
||||
|
|
|
|||
|
|
@ -5,9 +5,6 @@ from data.database import Image, ImageStorage, Repository
|
|||
from data import model
|
||||
from app import app, storage as store
|
||||
|
||||
import boto.s3.connection
|
||||
import boto.s3.key
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
|
@ -22,23 +19,9 @@ query = (Image
|
|||
.switch(Image)
|
||||
.join(Repository))
|
||||
|
||||
|
||||
bad_count = 0
|
||||
good_count = 0
|
||||
|
||||
s3_conn = boto.s3.connection.S3Connection(app.config['AWS_ACCESS_KEY'],
|
||||
app.config['AWS_SECRET_KEY'])
|
||||
s3_bucket = s3_conn.get_bucket('quay-registry')
|
||||
|
||||
PATHS = [
|
||||
store.image_json_path,
|
||||
store.image_checksum_path,
|
||||
store.image_layer_path,
|
||||
store.image_ancestry_path,
|
||||
store.image_file_trie_path,
|
||||
store.image_file_diffs_path,
|
||||
]
|
||||
|
||||
def resolve_or_create(repo, docker_image_id, new_ancestry):
|
||||
existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id)
|
||||
if existing:
|
||||
|
|
@ -58,42 +41,9 @@ def resolve_or_create(repo, docker_image_id, new_ancestry):
|
|||
logger.debug('Created image: %s' % created)
|
||||
return created
|
||||
except ImageStorage.DoesNotExist:
|
||||
logger.warning('No storage for ancestor, tring to find it anywhere: %s',
|
||||
docker_image_id)
|
||||
try:
|
||||
found = Image.get(docker_image_id=docker_image_id)
|
||||
logger.debug('Found some legacy storage for docker_image_id: %s',
|
||||
docker_image_id)
|
||||
new_storage = ImageStorage.create(checksum=found.checksum,
|
||||
created=found.created,
|
||||
comment=found.comment,
|
||||
command=found.command,
|
||||
image_size=found.image_size)
|
||||
|
||||
logger.debug('Migrating data to new storage: %s' % new_storage.uuid)
|
||||
|
||||
for path in PATHS:
|
||||
old_path = path(found.repository.namespace, found.repository.name,
|
||||
docker_image_id, None)
|
||||
new_path = path(None, None, None, new_storage.uuid)
|
||||
logger.debug('Copying %s -> %s', old_path, new_path)
|
||||
|
||||
old_path_key = s3_bucket.get_key(old_path)
|
||||
old_path_key.copy('quay-registry', new_path, encrypt_key=True,
|
||||
validate_dst_bucket=False)
|
||||
|
||||
logger.debug('Creating new image from copied legacy storage: %s',
|
||||
new_storage.uuid)
|
||||
created = Image.create(docker_image_id=docker_image_id,
|
||||
repository=repo,
|
||||
storage=new_storage, ancestors=new_ancestry)
|
||||
logger.debug('Created image: %s' % created)
|
||||
return created
|
||||
|
||||
except Image.DoesNotExist:
|
||||
msg = 'No image available anywhere for storage: %s in namespace: %s'
|
||||
logger.error(msg, docker_image_id, repo.namespace)
|
||||
raise RuntimeError()
|
||||
msg = 'No image available anywhere for storage: %s in namespace: %s'
|
||||
logger.error(msg, docker_image_id, repo.namespace)
|
||||
raise RuntimeError()
|
||||
|
||||
|
||||
def all_ancestors_exist(ancestors):
|
||||
|
|
@ -109,11 +59,7 @@ def all_ancestors_exist(ancestors):
|
|||
cant_fix = []
|
||||
for img in query:
|
||||
try:
|
||||
uuid = img.storage.uuid
|
||||
ancestry_storage = store.image_ancestry_path(img.repository.namespace,
|
||||
img.repository.name,
|
||||
img.docker_image_id,
|
||||
uuid)
|
||||
ancestry_storage = store.image_ancestry_path(img.storage.uuid)
|
||||
if store.exists(ancestry_storage):
|
||||
full_ancestry = json.loads(store.get_content(ancestry_storage))[1:]
|
||||
full_ancestry.reverse()
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
from data.database import Image, RepositoryTag, Repository
|
||||
|
||||
from app import storage as store
|
||||
|
||||
|
||||
tag_query = (RepositoryTag
|
||||
.select(RepositoryTag, Image, Repository)
|
||||
.join(Repository)
|
||||
.switch(RepositoryTag)
|
||||
.join(Image))
|
||||
|
||||
for tag in tag_query:
|
||||
if tag.image.repository.id != tag.repository.id:
|
||||
print('Repository tag pointing to external image: %s/%s:%s' %
|
||||
(tag.repository.namespace, tag.repository.name, tag.name))
|
||||
|
||||
proper_image_layer_path = store.image_layer_path(tag.repository.namespace,
|
||||
tag.repository.name,
|
||||
tag.image.docker_image_id)
|
||||
|
||||
has_storage = False
|
||||
if store.exists(proper_image_layer_path):
|
||||
print('Storage already in place: %s' % proper_image_layer_path)
|
||||
has_storage = True
|
||||
else:
|
||||
print('Storage missing: %s' % proper_image_layer_path)
|
||||
|
||||
has_db_entry = False
|
||||
new_image = None
|
||||
try:
|
||||
new_image = Image.get(Image.docker_image_id == tag.image.docker_image_id,
|
||||
Image.repository == tag.repository)
|
||||
has_db_entry = True
|
||||
print('DB image in place: %s invalid image id: %s' % (new_image.id,
|
||||
tag.image.id))
|
||||
except Image.DoesNotExist:
|
||||
print('DB image missing: %s' % tag.image.docker_image_id)
|
||||
|
||||
if has_storage and has_db_entry:
|
||||
print('Switching tag to proper image %s/%s/%s -> %s' %
|
||||
(tag.repository.namespace, tag.repository.name, tag.name,
|
||||
new_image.id))
|
||||
tag.image = new_image
|
||||
tag.save()
|
||||
print('Done')
|
||||
Reference in a new issue