diff --git a/data/database.py b/data/database.py index ffa8c909e..9ec53435b 100644 --- a/data/database.py +++ b/data/database.py @@ -228,12 +228,7 @@ class Image(BaseModel): # security reasons. So rather than Repository <-> Image being many to many # each image now belongs to exactly one repository. docker_image_id = CharField() - checksum = CharField(null=True) - created = DateTimeField(null=True) - comment = TextField(null=True) - command = TextField(null=True) repository = ForeignKeyField(Repository) - image_size = BigIntegerField(null=True) # '/' separated list of ancestory ids, e.g. /1/2/6/7/10/ ancestors = CharField(index=True, default='/', max_length=64535, null=True) @@ -244,7 +239,7 @@ class Image(BaseModel): database = db indexes = ( # we don't really want duplicates - (('repository', 'docker_image_id'), False), + (('repository', 'docker_image_id'), True), ) diff --git a/data/model/legacy.py b/data/model/legacy.py index f27e29170..c7fd49b4c 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1011,14 +1011,6 @@ def find_create_or_link_image(docker_image_id, repository, username, return new_image -def set_image_checksum(docker_image_id, repository, checksum): - fetched = Image.get(Image.docker_image_id == docker_image_id, - Image.repository == repository) - fetched.checksum = checksum - fetched.save() - return fetched - - def set_image_size(docker_image_id, namespace_name, repository_name, image_size): try: @@ -1122,15 +1114,9 @@ def garbage_collect_repository(namespace_name, repository_name): for image_id_to_remove in to_remove: image_to_remove = all_images[image_id_to_remove] - if image_to_remove.storage and image_to_remove.storage.id: - logger.debug('Adding image storage to the gc list: %s', - image_to_remove.storage.uuid) - uuids_to_check_for_gc.add(image_to_remove.storage.uuid) - else: - image_path = config.store.image_path(namespace_name, repository_name, - image_to_remove.docker_image_id, None) - logger.debug('Deleting image storage: %s', image_path) - config.store.remove(image_path) + logger.debug('Adding image storage to the gc list: %s', + image_to_remove.storage.uuid) + uuids_to_check_for_gc.add(image_to_remove.storage.uuid) image_to_remove.delete_instance() @@ -1145,8 +1131,7 @@ def garbage_collect_repository(namespace_name, repository_name): for storage in storage_to_remove: logger.debug('Garbage collecting image storage: %s', storage.uuid) storage.delete_instance() - image_path = config.store.image_path(namespace_name, repository_name, - image_to_remove.docker_image_id, storage.uuid) + image_path = config.store.image_path(storage.uuid) config.store.remove(image_path) return len(to_remove) diff --git a/endpoints/api/image.py b/endpoints/api/image.py index 4571b140b..6593d18df 100644 --- a/endpoints/api/image.py +++ b/endpoints/api/image.py @@ -79,8 +79,7 @@ class RepositoryImageChanges(RepositoryParamResource): if not image: raise NotFound() - uuid = image.storage and image.storage.uuid - diffs_path = store.image_file_diffs_path(namespace, repository, image_id, uuid) + diffs_path = store.image_file_diffs_path(image.storage.uuid) try: response_json = json.loads(store.get_content(diffs_path)) diff --git a/endpoints/index.py b/endpoints/index.py index b1a22f09e..f1d6075f7 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -181,7 +181,7 @@ def update_user(username): @generate_headers(role='write') def create_repository(namespace, repository): profile.debug('Parsing image descriptions') - image_descriptions = json.loads(request.data) + image_descriptions = json.loads(request.data.decode('utf8')) profile.debug('Looking up repository') repo = model.get_repository(namespace, repository) @@ -292,13 +292,11 @@ def update_images(namespace, repository): abort(404, message='Unknown repository', issue='unknown-repo') profile.debug('Parsing image data') - image_with_checksums = json.loads(request.data) + image_with_checksums = json.loads(request.data.decode('utf8')) updated_tags = {} for image in image_with_checksums: - profile.debug('Setting checksum for image id: %s to %s', image['id'], image['checksum']) updated_tags[image['Tag']] = image['id'] - model.set_image_checksum(image['id'], repo, image['checksum']) if get_authenticated_user(): profile.debug('Publishing push event') @@ -366,7 +364,7 @@ def get_repository_images(namespace, repository): for image in model.get_repository_images(namespace, repository): new_image_view = { 'id': image.docker_image_id, - 'checksum': image.checksum, + 'checksum': image.storage.checksum, } all_images.append(new_image_view) diff --git a/endpoints/registry.py b/endpoints/registry.py index 8af9fcb03..76c7d56cd 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -38,27 +38,29 @@ class SocketReader(object): return buf -def image_is_uploading(namespace, repository, image_id, repo_image): - if repo_image and repo_image.storage and repo_image.storage.uploading is not None: +def image_is_uploading(repo_image): + if repo_image is None: + return False + + if repo_image.storage.uploading is not None: return repo_image.storage.uploading - logger.warning('Setting legacy upload flag') - uuid = repo_image and repo_image.storage and repo_image.storage.uuid - mark_path = store.image_mark_path(namespace, repository, image_id, uuid) + logger.warning('Checking legacy upload flag') + mark_path = store.image_mark_path(repo_image.storage.uuid) return store.exists(mark_path) -def mark_upload_complete(namespace, repository, image_id, repo_image): - if repo_image and repo_image.storage and repo_image.storage.uploading is not None: - repo_image.storage.uploading = False - repo_image.storage.save() - else: +def set_uploading_flag(repo_image, is_image_uploading): + if repo_image.storage.uploading is None and not is_image_uploading: logger.warning('Removing legacy upload flag') - uuid = repo_image and repo_image.storage and repo_image.storage.uuid - mark_path = store.image_mark_path(namespace, repository, image_id, uuid) + uuid = repo_image.storage.uuid + mark_path = store.image_mark_path(uuid) if store.exists(mark_path): store.remove(mark_path) + repo_image.storage.uploading = is_image_uploading + repo_image.storage.save() + def require_completion(f): """This make sure that the image push correctly finished.""" @@ -66,7 +68,7 @@ def require_completion(f): def wrapper(namespace, repository, *args, **kwargs): image_id = kwargs['image_id'] repo_image = model.get_repo_image(namespace, repository, image_id) - if image_is_uploading(namespace, repository, image_id, repo_image): + if image_is_uploading(repo_image): abort(400, 'Image %(image_id)s is being uploaded, retry later', issue='upload-in-progress', image_id=kwargs['image_id']) @@ -111,21 +113,20 @@ def get_image_layer(namespace, repository, image_id, headers): profile.debug('Looking up repo image') repo_image = model.get_repo_image(namespace, repository, image_id) - uuid = repo_image and repo_image.storage and repo_image.storage.uuid - profile.debug('Looking up the layer path') - path = store.image_layer_path(namespace, repository, image_id, uuid) - - profile.debug('Looking up the direct download URL') - direct_download_url = store.get_direct_download_url(path) - - if direct_download_url: - profile.debug('Returning direct download URL') - return redirect(direct_download_url) try: + path = store.image_layer_path(repo_image.storage.uuid) + + profile.debug('Looking up the direct download URL') + direct_download_url = store.get_direct_download_url(path) + + if direct_download_url: + profile.debug('Returning direct download URL') + return redirect(direct_download_url) + profile.debug('Streaming layer data') return Response(store.stream_read(path), headers=headers) - except IOError: + except (IOError, AttributeError): profile.debug('Image not found') abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) @@ -144,21 +145,19 @@ def put_image_layer(namespace, repository, image_id): profile.debug('Retrieving image') repo_image = model.get_repo_image(namespace, repository, image_id) - - uuid = repo_image and repo_image.storage and repo_image.storage.uuid try: profile.debug('Retrieving image data') - json_data = store.get_content(store.image_json_path(namespace, repository, - image_id, uuid)) - except IOError: + uuid = repo_image.storage.uuid + json_data = store.get_content(store.image_json_path(uuid)) + except (IOError, AttributeError): abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) profile.debug('Retrieving image path info') - layer_path = store.image_layer_path(namespace, repository, image_id, uuid) + layer_path = store.image_layer_path(uuid) if (store.exists(layer_path) and not - image_is_uploading(namespace, repository, image_id, repo_image)): + image_is_uploading(repo_image)): abort(409, 'Image already exists', issue='image-exists', image_id=image_id) profile.debug('Storing layer data') @@ -193,9 +192,7 @@ def put_image_layer(namespace, repository, image_id): '{0}'.format(e)) try: - checksum = store.get_content(store.image_checksum_path(namespace, - repository, - image_id, uuid)) + checksum = store.get_content(store.image_checksum_path(uuid)) except IOError: # We don't have a checksum stored yet, that's fine skipping the check. # Not removing the mark though, image is not downloadable yet. @@ -209,7 +206,7 @@ def put_image_layer(namespace, repository, image_id): issue='checksum-mismatch', image_id=image_id) # Checksum is ok, we remove the marker - mark_upload_complete(namespace, repository, image_id, repo_image) + set_uploading_flag(repo_image, False) # The layer is ready for download, send a job to the work queue to # process it. @@ -232,9 +229,11 @@ def put_image_checksum(namespace, repository, image_id): if not permission.can(): abort(403) - checksum = request.headers.get('X-Docker-Checksum') + checksum = (request.headers.get('X-Docker-Checksum-Payload', None) or + request.headers.get('X-Docker-Checksum')) if not checksum: - abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum', image_id=image_id) + abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum', + image_id=image_id) if not session.get('checksum'): abort(400, 'Checksum not found in Cookie for image %(image_id)s', @@ -242,21 +241,19 @@ def put_image_checksum(namespace, repository, image_id): profile.debug('Looking up repo image') repo_image = model.get_repo_image(namespace, repository, image_id) - - uuid = repo_image and repo_image.storage and repo_image.storage.uuid + uuid = repo_image.storage.uuid profile.debug('Looking up repo layer data') - if not store.exists(store.image_json_path(namespace, repository, image_id, - uuid)): + if not store.exists(store.image_json_path(uuid)): abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) profile.debug('Marking image path') - if not image_is_uploading(namespace, repository, image_id, repo_image): + if not image_is_uploading(repo_image): abort(409, 'Cannot set checksum for image %(image_id)s', issue='image-write-error', image_id=image_id) profile.debug('Storing image checksum') - err = store_checksum(namespace, repository, image_id, uuid, checksum) + err = store_checksum(repo_image.storage, checksum) if err: abort(400, err) @@ -268,7 +265,7 @@ def put_image_checksum(namespace, repository, image_id): issue='checksum-mismatch', image_id=image_id) # Checksum is ok, we remove the marker - mark_upload_complete(namespace, repository, image_id, repo_image) + set_uploading_flag(repo_image, False) # The layer is ready for download, send a job to the work queue to # process it. @@ -296,13 +293,11 @@ def get_image_json(namespace, repository, image_id, headers): profile.debug('Looking up repo image') repo_image = model.get_repo_image(namespace, repository, image_id) - uuid = repo_image and repo_image.storage and repo_image.storage.uuid profile.debug('Looking up repo layer data') try: - data = store.get_content(store.image_json_path(namespace, repository, - image_id, uuid)) - except IOError: + data = store.get_content(store.image_json_path(repo_image.storage.uuid)) + except (IOError, AttributeError): flask_abort(404) profile.debug('Looking up repo layer size') @@ -312,12 +307,6 @@ def get_image_json(namespace, repository, image_id, headers): except OSError: pass - profile.debug('Retrieving checksum') - checksum_path = store.image_checksum_path(namespace, repository, image_id, - uuid) - if store.exists(checksum_path): - headers['X-Docker-Checksum'] = store.get_content(checksum_path) - response = make_response(data, 200) response.headers.extend(headers) return response @@ -337,13 +326,11 @@ def get_image_ancestry(namespace, repository, image_id, headers): profile.debug('Looking up repo image') repo_image = model.get_repo_image(namespace, repository, image_id) - uuid = repo_image and repo_image.storage and repo_image.storage.uuid profile.debug('Looking up image data') try: - data = store.get_content(store.image_ancestry_path(namespace, repository, - image_id, uuid)) - except IOError: + data = store.get_content(store.image_ancestry_path(repo_image.storage.uuid)) + except (IOError, AttributeError): abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) @@ -355,32 +342,32 @@ def get_image_ancestry(namespace, repository, image_id, headers): return response -def generate_ancestry(namespace, repository, image_id, uuid, parent_id=None, +def generate_ancestry(image_id, uuid, parent_id=None, parent_uuid=None): if not parent_id: - store.put_content(store.image_ancestry_path(namespace, repository, - image_id, uuid), + store.put_content(store.image_ancestry_path(uuid), json.dumps([image_id])) return - data = store.get_content(store.image_ancestry_path(namespace, repository, - parent_id, parent_uuid)) + data = store.get_content(store.image_ancestry_path(parent_uuid)) data = json.loads(data) data.insert(0, image_id) - store.put_content(store.image_ancestry_path(namespace, repository, - image_id, uuid), + store.put_content(store.image_ancestry_path(uuid), json.dumps(data)) -def store_checksum(namespace, repository, image_id, uuid, checksum): +def store_checksum(image_storage, checksum): checksum_parts = checksum.split(':') if len(checksum_parts) != 2: return 'Invalid checksum format' # We store the checksum - checksum_path = store.image_checksum_path(namespace, repository, image_id, - uuid) + checksum_path = store.image_checksum_path(image_storage.uuid) store.put_content(checksum_path, checksum) + # And store it in the db + image_storage.checksum = checksum + image_storage.save() + @registry.route('/images//json', methods=['PUT']) @process_auth @@ -393,9 +380,10 @@ def put_image_json(namespace, repository, image_id): profile.debug('Parsing image JSON') try: - data = json.loads(request.data) - except json.JSONDecodeError: + data = json.loads(request.data.decode('utf8')) + except ValueError: pass + if not data or not isinstance(data, dict): abort(400, 'Invalid JSON for image: %(image_id)s\nJSON: %(json)s', issue='invalid-request', image_id=image_id, json=request.data) @@ -406,22 +394,8 @@ def put_image_json(namespace, repository, image_id): profile.debug('Looking up repo image') repo_image = model.get_repo_image(namespace, repository, image_id) - uuid = repo_image and repo_image.storage and repo_image.storage.uuid + uuid = repo_image.storage.uuid - # Read the checksum - checksum = request.headers.get('X-Docker-Checksum') - if checksum: - # Storing the checksum is optional at this stage - profile.debug('Storing image checksum') - err = store_checksum(namespace, repository, image_id, uuid, checksum) - if err: - abort(400, err, issue='write-error') - - else: - # We cleanup any old checksum in case it's a retry after a fail - profile.debug('Cleanup old checksum') - store.remove(store.image_checksum_path(namespace, repository, image_id, - uuid)) if image_id != data['id']: abort(400, 'JSON data contains invalid id for image: %(image_id)s', issue='invalid-request', image_id=image_id) @@ -433,26 +407,33 @@ def put_image_json(namespace, repository, image_id): profile.debug('Looking up parent image') parent_image = model.get_repo_image(namespace, repository, parent_id) - parent_uuid = (parent_image and parent_image.storage and - parent_image.storage.uuid) + parent_uuid = parent_image and parent_image.storage.uuid if parent_id: profile.debug('Looking up parent image data') if (parent_id and not - store.exists(store.image_json_path(namespace, repository, parent_id, - parent_uuid))): + store.exists(store.image_json_path(parent_uuid))): abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s', issue='invalid-request', image_id=image_id, parent_id=parent_id) profile.debug('Looking up image storage paths') - json_path = store.image_json_path(namespace, repository, image_id, uuid) + json_path = store.image_json_path(uuid) profile.debug('Checking if image already exists') if (store.exists(json_path) and not - image_is_uploading(namespace, repository, image_id, repo_image)): + image_is_uploading(repo_image)): abort(409, 'Image already exists', issue='image-exists', image_id=image_id) + set_uploading_flag(repo_image, True) + + # We cleanup any old checksum in case it's a retry after a fail + profile.debug('Cleanup old checksum') + try: + store.remove(store.image_checksum_path(uuid)) + except Exception: + pass + # If we reach that point, it means that this is a new image or a retry # on a failed push # save the metadata @@ -468,8 +449,7 @@ def put_image_json(namespace, repository, image_id): store.put_content(json_path, request.data) profile.debug('Generating image ancestry') - generate_ancestry(namespace, repository, image_id, uuid, parent_id, - parent_uuid) + generate_ancestry(image_id, uuid, parent_id, parent_uuid) profile.debug('Done') return make_response('true', 200) @@ -479,12 +459,10 @@ def process_image_changes(namespace, repository, image_id): logger.debug('Generating diffs for image: %s' % image_id) repo_image = model.get_repo_image(namespace, repository, image_id) - uuid = repo_image and repo_image.storage and repo_image.storage.uuid + uuid = repo_image.storage.uuid - image_diffs_path = store.image_file_diffs_path(namespace, repository, - image_id, uuid) - image_trie_path = store.image_file_trie_path(namespace, repository, - image_id, uuid) + image_diffs_path = store.image_file_diffs_path(uuid) + image_trie_path = store.image_file_trie_path(uuid) if store.exists(image_diffs_path): logger.debug('Diffs already exist for image: %s' % image_id) @@ -506,7 +484,7 @@ def process_image_changes(namespace, repository, image_id): parent_trie.frombytes(parent_trie_bytes) # Read in the file entries from the layer tar file - layer_path = store.image_layer_path(namespace, repository, image_id, uuid) + layer_path = store.image_layer_path(uuid) with store.stream_read_file(layer_path) as layer_tar_stream: removed_files = set() layer_files = changes.files_and_dirs_from_tar(layer_tar_stream, diff --git a/initdb.py b/initdb.py index 33d2f048e..a48cac4e3 100644 --- a/initdb.py +++ b/initdb.py @@ -70,10 +70,10 @@ def __create_subtree(repo, structure, creator_username, parent): new_image = model.find_create_or_link_image(docker_image_id, repo, None, {}) new_image.storage.uuid = IMAGE_UUIDS[image_num % len(IMAGE_UUIDS)] + new_image.storage.uploading = False + new_image.storage.checksum = checksum new_image.storage.save() - model.set_image_checksum(docker_image_id, repo, checksum) - creation_time = REFERENCE_DATE + timedelta(days=image_num) command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)] command = json.dumps(command_list) if command_list else None @@ -85,9 +85,7 @@ def __create_subtree(repo, structure, creator_username, parent): random.randrange(1, 1024 * 1024 * 1024)) # Populate the diff file - diff_path = store.image_file_diffs_path(repo.namespace, repo.name, - docker_image_id, - new_image.storage.uuid) + diff_path = store.image_file_diffs_path(new_image.storage.uuid) source_diff = SAMPLE_DIFFS[image_num % len(SAMPLE_DIFFS)] with open(source_diff, 'r') as source_file: diff --git a/storage/basestorage.py b/storage/basestorage.py index 1e924ed1a..93b2b64e8 100644 --- a/storage/basestorage.py +++ b/storage/basestorage.py @@ -29,41 +29,35 @@ class BaseStorage(object): return tmpf, fn - def image_path(self, namespace, repository, image_id, storage_uuid): - if storage_uuid: - return '{0}/{1}/'.format(self.shared_images, storage_uuid) - else: - return '{0}/{1}/{2}/{3}/'.format(self.images, namespace, repository, - image_id) + def image_path(self, storage_uuid): + return '{0}/{1}/'.format(self.shared_images, storage_uuid) - def image_json_path(self, namespace, repository, image_id, storage_uuid): - base_path = self.image_path(namespace, repository, image_id, storage_uuid) + def image_json_path(self, storage_uuid): + base_path = self.image_path(storage_uuid) return '{0}json'.format(base_path) - def image_mark_path(self, namespace, repository, image_id, storage_uuid): - base_path = self.image_path(namespace, repository, image_id, storage_uuid) + def image_mark_path(self, storage_uuid): + base_path = self.image_path(storage_uuid) return '{0}_inprogress'.format(base_path) - def image_checksum_path(self, namespace, repository, image_id, storage_uuid): - base_path = self.image_path(namespace, repository, image_id, storage_uuid) + def image_checksum_path(self, storage_uuid): + base_path = self.image_path(storage_uuid) return '{0}_checksum'.format(base_path) - def image_layer_path(self, namespace, repository, image_id, storage_uuid): - base_path = self.image_path(namespace, repository, image_id, storage_uuid) + def image_layer_path(self, storage_uuid): + base_path = self.image_path(storage_uuid) return '{0}layer'.format(base_path) - def image_ancestry_path(self, namespace, repository, image_id, storage_uuid): - base_path = self.image_path(namespace, repository, image_id, storage_uuid) + def image_ancestry_path(self, storage_uuid): + base_path = self.image_path(storage_uuid) return '{0}ancestry'.format(base_path) - def image_file_trie_path(self, namespace, repository, image_id, - storage_uuid): - base_path = self.image_path(namespace, repository, image_id, storage_uuid) + def image_file_trie_path(self, storage_uuid): + base_path = self.image_path(storage_uuid) return '{0}files.trie'.format(base_path) - def image_file_diffs_path(self, namespace, repository, image_id, - storage_uuid): - base_path = self.image_path(namespace, repository, image_id, storage_uuid) + def image_file_diffs_path(self, storage_uuid): + base_path = self.image_path(storage_uuid) return '{0}diffs.json'.format(base_path) def get_direct_download_url(self, path, expires_in=60): diff --git a/test/data/test.db b/test/data/test.db index 801d3a28a..8c227a1ca 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/specs.py b/test/specs.py index c8f53a376..33db0493e 100644 --- a/test/specs.py +++ b/test/specs.py @@ -113,11 +113,11 @@ class IndexTestSpec(object): def build_index_specs(): return [ IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID), - PUBLIC_REPO, 200, 200, 200, 200), + PUBLIC_REPO, 404, 404, 404, 404), IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID), - PRIVATE_REPO), + PRIVATE_REPO, 403, 403, 404, 404), IndexTestSpec(url_for('registry.get_image_layer', image_id=FAKE_IMAGE_ID), - ORG_REPO), + ORG_REPO, 403, 403, 404, 404), IndexTestSpec(url_for('registry.put_image_layer', image_id=FAKE_IMAGE_ID), PUBLIC_REPO, 403, 403, 403, 403).set_method('PUT'), diff --git a/tools/auditancestry.py b/tools/auditancestry.py index c9c350a79..3bcd8aa78 100644 --- a/tools/auditancestry.py +++ b/tools/auditancestry.py @@ -5,9 +5,6 @@ from data.database import Image, ImageStorage, Repository from data import model from app import app, storage as store -import boto.s3.connection -import boto.s3.key - logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) @@ -22,23 +19,9 @@ query = (Image .switch(Image) .join(Repository)) - bad_count = 0 good_count = 0 -s3_conn = boto.s3.connection.S3Connection(app.config['AWS_ACCESS_KEY'], - app.config['AWS_SECRET_KEY']) -s3_bucket = s3_conn.get_bucket('quay-registry') - -PATHS = [ - store.image_json_path, - store.image_checksum_path, - store.image_layer_path, - store.image_ancestry_path, - store.image_file_trie_path, - store.image_file_diffs_path, -] - def resolve_or_create(repo, docker_image_id, new_ancestry): existing = model.get_repo_image(repo.namespace, repo.name, docker_image_id) if existing: @@ -58,42 +41,9 @@ def resolve_or_create(repo, docker_image_id, new_ancestry): logger.debug('Created image: %s' % created) return created except ImageStorage.DoesNotExist: - logger.warning('No storage for ancestor, tring to find it anywhere: %s', - docker_image_id) - try: - found = Image.get(docker_image_id=docker_image_id) - logger.debug('Found some legacy storage for docker_image_id: %s', - docker_image_id) - new_storage = ImageStorage.create(checksum=found.checksum, - created=found.created, - comment=found.comment, - command=found.command, - image_size=found.image_size) - - logger.debug('Migrating data to new storage: %s' % new_storage.uuid) - - for path in PATHS: - old_path = path(found.repository.namespace, found.repository.name, - docker_image_id, None) - new_path = path(None, None, None, new_storage.uuid) - logger.debug('Copying %s -> %s', old_path, new_path) - - old_path_key = s3_bucket.get_key(old_path) - old_path_key.copy('quay-registry', new_path, encrypt_key=True, - validate_dst_bucket=False) - - logger.debug('Creating new image from copied legacy storage: %s', - new_storage.uuid) - created = Image.create(docker_image_id=docker_image_id, - repository=repo, - storage=new_storage, ancestors=new_ancestry) - logger.debug('Created image: %s' % created) - return created - - except Image.DoesNotExist: - msg = 'No image available anywhere for storage: %s in namespace: %s' - logger.error(msg, docker_image_id, repo.namespace) - raise RuntimeError() + msg = 'No image available anywhere for storage: %s in namespace: %s' + logger.error(msg, docker_image_id, repo.namespace) + raise RuntimeError() def all_ancestors_exist(ancestors): @@ -109,11 +59,7 @@ def all_ancestors_exist(ancestors): cant_fix = [] for img in query: try: - uuid = img.storage.uuid - ancestry_storage = store.image_ancestry_path(img.repository.namespace, - img.repository.name, - img.docker_image_id, - uuid) + ancestry_storage = store.image_ancestry_path(img.storage.uuid) if store.exists(ancestry_storage): full_ancestry = json.loads(store.get_content(ancestry_storage))[1:] full_ancestry.reverse() diff --git a/tools/audittagimages.py b/tools/audittagimages.py deleted file mode 100644 index 7ae223146..000000000 --- a/tools/audittagimages.py +++ /dev/null @@ -1,45 +0,0 @@ -from data.database import Image, RepositoryTag, Repository - -from app import storage as store - - -tag_query = (RepositoryTag - .select(RepositoryTag, Image, Repository) - .join(Repository) - .switch(RepositoryTag) - .join(Image)) - -for tag in tag_query: - if tag.image.repository.id != tag.repository.id: - print('Repository tag pointing to external image: %s/%s:%s' % - (tag.repository.namespace, tag.repository.name, tag.name)) - - proper_image_layer_path = store.image_layer_path(tag.repository.namespace, - tag.repository.name, - tag.image.docker_image_id) - - has_storage = False - if store.exists(proper_image_layer_path): - print('Storage already in place: %s' % proper_image_layer_path) - has_storage = True - else: - print('Storage missing: %s' % proper_image_layer_path) - - has_db_entry = False - new_image = None - try: - new_image = Image.get(Image.docker_image_id == tag.image.docker_image_id, - Image.repository == tag.repository) - has_db_entry = True - print('DB image in place: %s invalid image id: %s' % (new_image.id, - tag.image.id)) - except Image.DoesNotExist: - print('DB image missing: %s' % tag.image.docker_image_id) - - if has_storage and has_db_entry: - print('Switching tag to proper image %s/%s/%s -> %s' % - (tag.repository.namespace, tag.repository.name, tag.name, - new_image.id)) - tag.image = new_image - tag.save() - print('Done')