From ea18790dfedc33a25f1c42c1c307d29287800a94 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 12 Jul 2016 16:09:13 -0400 Subject: [PATCH] Get V1 registry code working with new model methods --- data/model/image.py | 11 +++ data/model/v1.py | 162 ++++++++++++++++++++++++++------------- endpoints/v1/registry.py | 12 +-- 3 files changed, 124 insertions(+), 61 deletions(-) diff --git a/data/model/image.py b/data/model/image.py index 031f4a660..b636c64fe 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -99,6 +99,17 @@ def get_repo_image_extended(namespace_name, repository_name, docker_image_id): return images[0] +def get_repo_image_and_storage(namespace_name, repository_name, docker_image_id): + def limit_to_image_id(query): + return query.where(Image.docker_image_id == docker_image_id) + + images = _get_repository_images_and_storages(namespace_name, repository_name, limit_to_image_id) + if not images: + return None + + return images[0] + + def _get_repository_images_and_storages(namespace_name, repository_name, query_modifier): query = (Image .select(Image, ImageStorage) diff --git a/data/model/v1.py b/data/model/v1.py index fd934b3df..3dfa1e123 100644 --- a/data/model/v1.py +++ b/data/model/v1.py @@ -1,26 +1,36 @@ from app import app, storage as store from data import model +from data.model import db_transaction from util.morecollections import AttrDict - -# TODO(jzelinskie): implement all of these methods using both legacy and new models. - def placement_locations_docker_v1(namespace_name, repo_name, image_id): - repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id) - if repo_image is None: + """ Returns all the placements for the image with the given V1 Docker ID, found under the + given repository or None if no image was found. + """ + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: return None + return repo_image.storage.locations def placement_locations_and_path_docker_v1(namespace_name, repo_name, image_id): + """ Returns a tuple of the placements and storage path location for the image with the + given V1 Docker ID, found under the given repository or None if no image was found. + """ repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id) - if not repo_image: + if not repo_image or repo_image.storage is None: return None, None - return model.storage.get_layer_path(repo_image.storage), repo_image.storage.locations + + return repo_image.storage.locations, model.storage.get_layer_path(repo_image.storage) def docker_v1_metadata(namespace_name, repo_name, image_id): - if not repo_image: + """ Returns various pieces of metadata associated with an image with the given V1 Docker ID, + including the checksum and its V1 JSON metadata. + """ + repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) + if repo_image is None: return None return AttrDict({ @@ -34,113 +44,155 @@ def docker_v1_metadata(namespace_name, repo_name, image_id): def update_docker_v1_metadata(namespace_name, repo_name, image_id, created_date_str, comment, command, compat_json, parent_image_id=None): - # Old implementation: - # parent_image = get_repo_extended(namespace_name, repo_name, parent_image_id) - # model.image.set_image_metadata(image_id, namespace_name, repo_name, create_date_str, comment, command, compat_json, parent_image) - pass + """ Updates various pieces of V1 metadata associated with a particular image. """ + parent_image = None + if parent_image_id is not None: + parent_image = model.image.get_repo_image(namespace_name, repo_name, parent_image_id) + + model.image.set_image_metadata(image_id, namespace_name, repo_name, created_date_str, comment, + command, compat_json, parent=parent_image) def storage_exists(namespace_name, repo_name, image_id): - repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id) - try: - layer_path = store.v1_image_layer_path(repo_image.storage.uuid) - except AttributeError: + """ Returns whether storage already exists for the image with the V1 Docker ID under the + given repository. + """ + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: return False - if (store.exists(repo_image.storage.locations, layer_path) and not - repo_image.storage.uploading): - return True - return False + if repo_image.storage.uploading: + return False + + layer_path = model.storage.get_layer_path(repo_image.storage) + return store.exists(repo_image.storage.locations, layer_path) -def store_docker_v1_checksum(namespace_name, repo_name, image_id, checksum, content_checksum): - ## Old implementation: - # UPDATE repo_image.storage.content_checksum = content_checksum - # UPDATE repo_image.v1_checksum = checksum - pass +def store_docker_v1_checksums(namespace_name, repo_name, image_id, checksum, content_checksum): + """ Stores the various V1 checksums for the image with the V1 Docker ID. """ + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return + + with db_transaction(): + repo_image.storage.content_checksum = content_checksum + repo_image.v1_checksum = checksum + + repo_image.storage.save() + repo_image.save() def is_image_uploading(namespace_name, repo_name, image_id): - repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id) - if repo_image is None: + """ Returns whether the image with the V1 Docker ID is currently marked as uploading. """ + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: return False + return repo_image.storage.uploading def update_image_uploading(namespace_name, repo_name, image_id, is_uploading): - ## Old implementation: - # UPDATE repo_image.storage.uploading = is_uploading - pass + """ Marks the image with the V1 Docker ID with the given uploading status. """ + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return + + repo_image.storage.uploading = is_uploading + repo_image.storage.save() + return repo_image.storage def update_image_sizes(namespace_name, repo_name, image_id, size, uncompressed_size): - model.storage.set_image_storage_metadata( - image_id, - namespace_name, - repo_name, - size, - uncompressed_size, - ) + """ Updates the sizing information for the image with the given V1 Docker ID. """ + model.storage.set_image_storage_metadata(image_id, namespace_name, repo_name, size, + uncompressed_size) def get_image_size(namespace_name, repo_name, image_id): + """ Returns the wire size of the image with the given Docker V1 ID. """ + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return None + return repo_image.storage.image_size def create_bittorrent_pieces(namespace_name, repo_name, image_id, pieces_bytes): - repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id) - try: - model.storage.save_torrent_info( - repo_image.storage, - app.config['BITTORRENT_PIECE_SIZE'], - pieces_bytes - ) - except AttributeError: - pass + """ Saves the bittorrent piece hashes for the image with the given Docker V1 ID. """ + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return + + model.storage.save_torrent_info(repo_image.storage, app.config['BITTORRENT_PIECE_SIZE'], + pieces_bytes) def image_ancestry(namespace_name, repo_name, image_id): + """ Returns a list containing the full ancestry of Docker V1 IDs, in order, for the image with + the givne Docker V1 ID. + """ try: - image = model.image.get_image_by_id(namespace, repository, image_id) + image = model.image.get_image_by_id(namespace_name, repo_name, image_id) except model.InvalidImageException: return None - parents = model.image.get_parent_images(namespace, repository, image) + parents = model.image.get_parent_images(namespace_name, repo_name, image) ancestry_docker_ids = [image.docker_image_id] ancestry_docker_ids.extend([parent.docker_image_id for parent in parents]) + return ancestry_docker_ids def repository_exists(namespace_name, repo_name): + """ Returns whether the repository with the given name and namespace exists. """ repo = model.repository.get_repository(namespace_name, repo_name) return repo is not None -def create_or_link_image(username, repo_name, image_id, storage_location): - pass +def create_or_link_image(username, namespace_name, repo_name, image_id, storage_location): + """ Adds the given image to the given repository, by either linking to an existing image + visible to the user with the given username, or creating a new one if no existing image + matches. + """ + repo = model.repository.get_repository(namespace_name, repo_name) + model.image.find_create_or_link_image(image_id, repo, username, {}, storage_location) -def create_temp_hidden_tag(namespace_name, repo_name, expiration): - # was this code: - # model.tag.create_temporary_hidden_tag(repo, repo_image, - # app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']) - pass +def create_temp_hidden_tag(namespace_name, repo_name, image_id, expiration): + """ Creates a hidden tag under the matching namespace pointing to the image with the given V1 + Docker ID. + """ + repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) + if repo_image is None: + return + + repo = repo_image.repository + model.tag.create_temporary_hidden_tag(repo, repo_image, expiration) def list_tags(namespace_name, repo_name): + """ Returns all the tags defined in the repository with the given namespace and name. """ return model.tag.list_repository_tags(namespace_name, repo_name) def create_or_update_tag(namespace_name, repo_name, image_id, tag_name): + """ Creates or updates a tag under the matching repository to point to the image with the given + Docker V1 ID. + """ model.tag.create_or_update_tag(namespace_name, repo_name, tag_name, image_id) def find_image_id_by_tag(namespace_name, repo_name, tag_name): + """ Returns the Docker V1 image ID for the HEAD image for the tag with the given name under + the matching repository, or None if none. + """ try: tag_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name) except model.DataModelException: return None + return tag_image.docker_image_id def delete_tag(namespace_name, repo_name, tag_name): + """ Deletes the given tag from the given repository. """ model.tag.delete_tag(namespace_name, repo_name, tag_name) diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py index 99c2e8f5e..6132c44b8 100644 --- a/endpoints/v1/registry.py +++ b/endpoints/v1/registry.py @@ -30,11 +30,10 @@ logger = logging.getLogger(__name__) def _finish_image(namespace, repository, image_id): # Checksum is ok, we remove the marker - v1.update_image_uploading(namespace, repository, image_id, False) + blob_ref = v1.update_image_uploading(namespace, repository, image_id, False) # Send a job to the work queue to replicate the image layer. - # TODO(jzelinskie): make this not use imagestorage - queue_storage_replication(namespace, repo_image.storage) + queue_storage_replication(namespace, blob_ref) def require_completion(f): @@ -292,7 +291,7 @@ def put_image_checksum(namespace, repository, image_id): if len(checksum_parts) != 2: abort(400, 'Invalid checksum format') - v1.store_docker_v1_checksum(namespace, repository, image_id, checksum, content_checksum) + v1.store_docker_v1_checksums(namespace, repository, image_id, checksum, content_checksum) if checksum not in session.get('checksum', []): logger.debug('session checksums: %s', session.get('checksum', [])) @@ -400,12 +399,13 @@ def put_image_json(namespace, repository, image_id): username = get_granted_username() logger.debug('Image not found, creating or linking image with initiating user context: %s', username) - v1.create_or_link_image(username, repository, image_id, store.preferred_locations[0]) + v1.create_or_link_image(username, namespace, repository, image_id, store.preferred_locations[0]) v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) # Create a temporary tag to prevent this image from getting garbage collected while the push # is in progress. - v1.create_temp_hidden_tag(namespace, repository, app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']) + v1.create_temp_hidden_tag(namespace, repository, image_id, + app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']) parent_id = data.get('parent', None) if parent_id: