From c06d395f964e89cfe02656581f133e9a68699245 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 30 Aug 2016 15:05:15 -0400 Subject: [PATCH] create interfaces for v1 and v2 data model --- data/interfaces/v1.py | 620 ++++++++++++++++++++----------- data/interfaces/v2.py | 777 ++++++++++++++++++++++++--------------- endpoints/v1/index.py | 23 +- endpoints/v1/registry.py | 70 ++-- endpoints/v1/tag.py | 17 +- endpoints/v2/blob.py | 34 +- endpoints/v2/catalog.py | 4 +- endpoints/v2/manifest.py | 38 +- endpoints/v2/tag.py | 6 +- endpoints/v2/v2auth.py | 8 +- image/__init__.py | 111 ------ image/appc/__init__.py | 2 +- image/common.py | 68 ++++ image/docker/squashed.py | 2 +- 14 files changed, 1048 insertions(+), 732 deletions(-) create mode 100644 image/common.py diff --git a/data/interfaces/v1.py b/data/interfaces/v1.py index cbffaa3ef..214ffee2c 100644 --- a/data/interfaces/v1.py +++ b/data/interfaces/v1.py @@ -1,263 +1,435 @@ +from collections import namedtuple + from app import app, storage as store from data import model from data.model import db_transaction from util.morecollections import AttrDict -from data.interfaces.common import repository_for_repo -def placement_locations_docker_v1(namespace_name, repo_name, image_id): - """ Returns all the placements for the image with the given V1 Docker ID, found under the - given repository or None if no image was found. + +class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', + 'is_public'])): """ - repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) - if repo_image is None or repo_image.storage is None: - return None - - return repo_image.storage.locations - - -def placement_locations_and_path_docker_v1(namespace_name, repo_name, image_id): - """ Returns a tuple of the placements and storage path location for the image with the - given V1 Docker ID, found under the given repository or None if no image was found. + Repository represents a namespaced collection of tags. """ - repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id) - if not repo_image or repo_image.storage is None: - return None, None - - return repo_image.storage.locations, model.storage.get_layer_path(repo_image.storage) -def docker_v1_metadata(namespace_name, repo_name, image_id): - """ Returns various pieces of metadata associated with an image with the given V1 Docker ID, - including the checksum and its V1 JSON metadata. +def _repository_for_repo(repo): """ - repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) - if repo_image is None: - return None - - return AttrDict({ - 'namespace_name': namespace_name, - 'repo_name': repo_name, - 'image_id': image_id, - 'checksum': repo_image.v1_checksum, - 'compat_json': repo_image.v1_json_metadata, - }) - - -def update_docker_v1_metadata(namespace_name, repo_name, image_id, created_date_str, comment, - command, compat_json, parent_image_id=None): - """ Updates various pieces of V1 metadata associated with a particular image. """ - parent_image = None - if parent_image_id is not None: - parent_image = model.image.get_repo_image(namespace_name, repo_name, parent_image_id) - - model.image.set_image_metadata(image_id, namespace_name, repo_name, created_date_str, comment, - command, compat_json, parent=parent_image) - - -def storage_exists(namespace_name, repo_name, image_id): - """ Returns whether storage already exists for the image with the V1 Docker ID under the - given repository. + Returns a Repository object representing the repo data model instance given. """ - repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) - if repo_image is None or repo_image.storage is None: - return False - - if repo_image.storage.uploading: - return False - - layer_path = model.storage.get_layer_path(repo_image.storage) - return store.exists(repo_image.storage.locations, layer_path) + return Repository( + id=repo.id, + name=repo.name, + namespace_name=repo.namespace_user.username, + description=repo.description, + is_public=model.repository.is_repository_public(repo) + ) -def store_docker_v1_checksums(namespace_name, repo_name, image_id, checksum, content_checksum): - """ Stores the various V1 checksums for the image with the V1 Docker ID. """ - repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) - if repo_image is None or repo_image.storage is None: - return +class DockerRegistryV1DataInterface(object): + """ + Interface that represents all data store interactions required by a Docker Registry v1. + """ - with db_transaction(): - repo_image.storage.content_checksum = content_checksum - repo_image.v1_checksum = checksum + @classmethod + def placement_locations_docker_v1(cls, namespace_name, repo_name, image_id): + """ + Returns all the placements for the image with the given V1 Docker ID, found under the given + repository or None if no image was found. + """ + raise NotImplementedError() + @classmethod + def placement_locations_and_path_docker_v1(cls, namespace_name, repo_name, image_id): + """ + Returns all the placements for the image with the given V1 Docker ID, found under the given + repository or None if no image was found. + """ + raise NotImplementedError() + + @classmethod + def docker_v1_metadata(cls, namespace_name, repo_name, image_id): + """ + Returns various pieces of metadata associated with an image with the given V1 Docker ID, + including the checksum and its V1 JSON metadata. + """ + raise NotImplementedError() + + @classmethod + def update_docker_v1_metadata(cls, namespace_name, repo_name, image_id, created_date_str, comment, + command, compat_json, parent_image_id=None): + """ + Updates various pieces of V1 metadata associated with a particular image. + """ + raise NotImplementedError() + + @classmethod + def storage_exists(cls, namespace_name, repo_name, image_id): + """ + Returns whether storage already exists for the image with the V1 Docker ID under the given + repository. + """ + raise NotImplementedError() + + @classmethod + def store_docker_v1_checksums(cls, namespace_name, repo_name, image_id, checksum, content_checksum): + """ + Stores the various V1 checksums for the image with the V1 Docker ID. + """ + raise NotImplementedError() + + @classmethod + def is_image_uploading(cls, namespace_name, repo_name, image_id): + """ + Returns whether the image with the V1 Docker ID is currently marked as uploading. + """ + raise NotImplementedError() + + @classmethod + def update_image_uploading(cls, namespace_name, repo_name, image_id, is_uploading): + """ Marks the image with the V1 Docker ID with the given uploading status. """ + raise NotImplementedError() + + @classmethod + def update_image_sizes(cls, namespace_name, repo_name, image_id, size, uncompressed_size): + """ + Updates the sizing information for the image with the given V1 Docker ID. + """ + raise NotImplementedError() + + @classmethod + def get_image_size(cls, namespace_name, repo_name, image_id): + """ + Returns the wire size of the image with the given Docker V1 ID. + """ + raise NotImplementedError() + + @classmethod + def create_bittorrent_pieces(cls, namespace_name, repo_name, image_id, pieces_bytes): + """ + Saves the BitTorrent piece hashes for the image with the given Docker V1 ID. + """ + raise NotImplementedError() + + @classmethod + def image_ancestry(cls, namespace_name, repo_name, image_id): + """ + Returns a list containing the full ancestry of Docker V1 IDs, in order, for the image with the + given Docker V1 ID. + """ + raise NotImplementedError() + + @classmethod + def repository_exists(cls, namespace_name, repo_name): + """ + Returns whether the repository with the given name and namespace exists. + """ + raise NotImplementedError() + + @classmethod + def create_or_link_image(cls, username, namespace_name, repo_name, image_id, storage_location): + """ + Adds the given image to the given repository, by either linking to an existing image visible to + the user with the given username, or creating a new one if no existing image matches. + """ + raise NotImplementedError() + + @classmethod + def create_temp_hidden_tag(cls, namespace_name, repo_name, image_id, expiration): + """ + Creates a hidden tag under the matching namespace pointing to the image with the given V1 Docker + ID. + """ + raise NotImplementedError() + + @classmethod + def list_tags(cls, namespace_name, repo_name): + """ + Returns all the tags defined in the repository with the given namespace and name. + """ + raise NotImplementedError() + + @classmethod + def create_or_update_tag(cls, namespace_name, repo_name, image_id, tag_name): + """ + Creates or updates a tag under the matching repository to point to the image with the given + Docker V1 ID. + """ + raise NotImplementedError() + + @classmethod + def find_image_id_by_tag(cls, namespace_name, repo_name, tag_name): + """ + Returns the Docker V1 image ID for the HEAD image for the tag with the given name under the + matching repository, or None if none. + """ + raise NotImplementedError() + + @classmethod + def delete_tag(cls, namespace_name, repo_name, tag_name): + """ Deletes the given tag from the given repository. """ + raise NotImplementedError() + + @classmethod + def load_token(cls, token): + """ + Loads the data associated with the given (deprecated) access token, and, if + found returns True. + """ + raise NotImplementedError() + + @classmethod + def verify_robot(cls, username, token): + """ + Returns True if the given robot username and token match an existing robot + account. + """ + raise NotImplementedError() + + @classmethod + def change_user_password(cls, user, new_password): + """ + Changes the password associated with the given user. + """ + raise NotImplementedError() + + @classmethod + def get_repository(cls, namespace_name, repo_name): + """ + Returns the repository with the given name under the given namespace or None + if none. + """ + raise NotImplementedError() + + @classmethod + def create_repository(cls, namespace_name, repo_name, user=None): + """ + Creates a new repository under the given namespace with the given name, for + the given user. + """ + raise NotImplementedError() + + @classmethod + def repository_is_public(cls, namespace_name, repo_name): + """ + Returns whether the repository with the given name under the given namespace + is public. If no matching repository was found, returns False. + """ + raise NotImplementedError() + + @classmethod + def validate_oauth_token(cls, token): + """ Returns whether the given OAuth token validates. """ + raise NotImplementedError() + + @classmethod + def get_sorted_matching_repositories(cls, search_term, only_public, can_read, limit): + """ + Returns a sorted list of repositories matching the given search term. + can_read is a callback that will be invoked for each repository found, to + filter results to only those visible to the current user (if any). + """ + raise NotImplementedError() + + + +class PreOCIModel(DockerRegistryV1DataInterface): + """ + PreOCIModel implements the data model for the v1 Docker Registry protocol using a database schema + before it was changed to support the OCI specification. + """ + @classmethod + def placement_locations_docker_v1(cls, namespace_name, repo_name, image_id): + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return None + return repo_image.storage.locations + + @classmethod + def placement_locations_and_path_docker_v1(cls, namespace_name, repo_name, image_id): + repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id) + if not repo_image or repo_image.storage is None: + return None, None + return repo_image.storage.locations, model.storage.get_layer_path(repo_image.storage) + + @classmethod + def docker_v1_metadata(cls, namespace_name, repo_name, image_id): + repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) + if repo_image is None: + return None + + return AttrDict({ + 'namespace_name': namespace_name, + 'repo_name': repo_name, + 'image_id': image_id, + 'checksum': repo_image.v1_checksum, + 'compat_json': repo_image.v1_json_metadata, + }) + + @classmethod + def update_docker_v1_metadata(cls, namespace_name, repo_name, image_id, created_date_str, comment, + command, compat_json, parent_image_id=None): + parent_image = None + if parent_image_id is not None: + parent_image = model.image.get_repo_image(namespace_name, repo_name, parent_image_id) + + model.image.set_image_metadata(image_id, namespace_name, repo_name, created_date_str, comment, + command, compat_json, parent=parent_image) + + @classmethod + def storage_exists(cls, namespace_name, repo_name, image_id): + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return False + + if repo_image.storage.uploading: + return False + + layer_path = model.storage.get_layer_path(repo_image.storage) + return store.exists(repo_image.storage.locations, layer_path) + + @classmethod + def store_docker_v1_checksums(cls, namespace_name, repo_name, image_id, checksum, content_checksum): + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return + + with db_transaction(): + repo_image.storage.content_checksum = content_checksum + repo_image.v1_checksum = checksum + repo_image.storage.save() + repo_image.save() + + @classmethod + def is_image_uploading(cls, namespace_name, repo_name, image_id): + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return False + return repo_image.storage.uploading + + @classmethod + def update_image_uploading(cls, namespace_name, repo_name, image_id, is_uploading): + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return + + repo_image.storage.uploading = is_uploading repo_image.storage.save() - repo_image.save() + return repo_image.storage + @classmethod + def update_image_sizes(cls, namespace_name, repo_name, image_id, size, uncompressed_size): + model.storage.set_image_storage_metadata(image_id, namespace_name, repo_name, size, + uncompressed_size) -def is_image_uploading(namespace_name, repo_name, image_id): - """ Returns whether the image with the V1 Docker ID is currently marked as uploading. """ - repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) - if repo_image is None or repo_image.storage is None: - return False + @classmethod + def get_image_size(cls, namespace_name, repo_name, image_id): + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return None + return repo_image.storage.image_size - return repo_image.storage.uploading + @classmethod + def create_bittorrent_pieces(cls, namespace_name, repo_name, image_id, pieces_bytes): + repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) + if repo_image is None or repo_image.storage is None: + return + model.storage.save_torrent_info(repo_image.storage, app.config['BITTORRENT_PIECE_SIZE'], + pieces_bytes) -def update_image_uploading(namespace_name, repo_name, image_id, is_uploading): - """ Marks the image with the V1 Docker ID with the given uploading status. """ - repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) - if repo_image is None or repo_image.storage is None: - return + @classmethod + def image_ancestry(cls, namespace_name, repo_name, image_id): + try: + image = model.image.get_image_by_id(namespace_name, repo_name, image_id) + except model.InvalidImageException: + return None - repo_image.storage.uploading = is_uploading - repo_image.storage.save() - return repo_image.storage + parents = model.image.get_parent_images(namespace_name, repo_name, image) + ancestry_docker_ids = [image.docker_image_id] + ancestry_docker_ids.extend([parent.docker_image_id for parent in parents]) + return ancestry_docker_ids + @classmethod + def repository_exists(cls, namespace_name, repo_name): + repo = model.repository.get_repository(namespace_name, repo_name) + return repo is not None -def update_image_sizes(namespace_name, repo_name, image_id, size, uncompressed_size): - """ Updates the sizing information for the image with the given V1 Docker ID. """ - model.storage.set_image_storage_metadata(image_id, namespace_name, repo_name, size, - uncompressed_size) + @classmethod + def create_or_link_image(cls, username, namespace_name, repo_name, image_id, storage_location): + repo = model.repository.get_repository(namespace_name, repo_name) + model.image.find_create_or_link_image(image_id, repo, username, {}, storage_location) + @classmethod + def create_temp_hidden_tag(cls, namespace_name, repo_name, image_id, expiration): + repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) + if repo_image is None: + return -def get_image_size(namespace_name, repo_name, image_id): - """ Returns the wire size of the image with the given Docker V1 ID. """ - repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) - if repo_image is None or repo_image.storage is None: - return None + repo = repo_image.repository + model.tag.create_temporary_hidden_tag(repo, repo_image, expiration) - return repo_image.storage.image_size + @classmethod + def list_tags(cls, namespace_name, repo_name): + return model.tag.list_repository_tags(namespace_name, repo_name) + @classmethod + def create_or_update_tag(cls, namespace_name, repo_name, image_id, tag_name): + model.tag.create_or_update_tag(namespace_name, repo_name, tag_name, image_id) -def create_bittorrent_pieces(namespace_name, repo_name, image_id, pieces_bytes): - """ Saves the bittorrent piece hashes for the image with the given Docker V1 ID. """ - repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) - if repo_image is None or repo_image.storage is None: - return + @classmethod + def find_image_id_by_tag(cls, namespace_name, repo_name, tag_name): + try: + tag_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name) + except model.DataModelException: + return None - model.storage.save_torrent_info(repo_image.storage, app.config['BITTORRENT_PIECE_SIZE'], - pieces_bytes) + return tag_image.docker_image_id + @classmethod + def delete_tag(cls, namespace_name, repo_name, tag_name): + model.tag.delete_tag(namespace_name, repo_name, tag_name) -def image_ancestry(namespace_name, repo_name, image_id): - """ Returns a list containing the full ancestry of Docker V1 IDs, in order, for the image with - the givne Docker V1 ID. - """ - try: - image = model.image.get_image_by_id(namespace_name, repo_name, image_id) - except model.InvalidImageException: - return None + @classmethod + def load_token(cls, token): + try: + model.token.load_token_data(token) + return True + except model.InvalidTokenException: + return False - parents = model.image.get_parent_images(namespace_name, repo_name, image) - ancestry_docker_ids = [image.docker_image_id] - ancestry_docker_ids.extend([parent.docker_image_id for parent in parents]) - return ancestry_docker_ids + @classmethod + def verify_robot(cls, username, token): + try: + return bool(model.user.verify_robot(username, token)) + except model.InvalidRobotException: + return False + @classmethod + def change_user_password(cls, user, new_password): + model.user.change_password(user, new_password) -def repository_exists(namespace_name, repo_name): - """ Returns whether the repository with the given name and namespace exists. """ - repo = model.repository.get_repository(namespace_name, repo_name) - return repo is not None + @classmethod + def get_repository(cls, namespace_name, repo_name): + repo = model.repository.get_repository(namespace_name, repo_name) + if repo is None: + return None + return _repository_for_repo(repo) + @classmethod + def create_repository(cls, namespace_name, repo_name, user=None): + model.repository.create_repository(namespace_name, repo_name, user) -def create_or_link_image(username, namespace_name, repo_name, image_id, storage_location): - """ Adds the given image to the given repository, by either linking to an existing image - visible to the user with the given username, or creating a new one if no existing image - matches. - """ - repo = model.repository.get_repository(namespace_name, repo_name) - model.image.find_create_or_link_image(image_id, repo, username, {}, storage_location) + @classmethod + def repository_is_public(cls, namespace_name, repo_name): + return model.repository.repository_is_public(namespace_name, repo_name) + @classmethod + def validate_oauth_token(cls, token): + return bool(model.oauth.validate_access_token(token)) -def create_temp_hidden_tag(namespace_name, repo_name, image_id, expiration): - """ Creates a hidden tag under the matching namespace pointing to the image with the given V1 - Docker ID. - """ - repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) - if repo_image is None: - return - - repo = repo_image.repository - model.tag.create_temporary_hidden_tag(repo, repo_image, expiration) - - -def list_tags(namespace_name, repo_name): - """ Returns all the tags defined in the repository with the given namespace and name. """ - return model.tag.list_repository_tags(namespace_name, repo_name) - - -def create_or_update_tag(namespace_name, repo_name, image_id, tag_name): - """ Creates or updates a tag under the matching repository to point to the image with the given - Docker V1 ID. - """ - model.tag.create_or_update_tag(namespace_name, repo_name, tag_name, image_id) - - -def find_image_id_by_tag(namespace_name, repo_name, tag_name): - """ Returns the Docker V1 image ID for the HEAD image for the tag with the given name under - the matching repository, or None if none. - """ - try: - tag_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name) - except model.DataModelException: - return None - - return tag_image.docker_image_id - - -def delete_tag(namespace_name, repo_name, tag_name): - """ Deletes the given tag from the given repository. """ - model.tag.delete_tag(namespace_name, repo_name, tag_name) - - -def load_token(token): - """ Loads the data associated with the given (deprecated) access token, and, if found - returns True. - """ - try: - model.token.load_token_data(token) - return True - except model.InvalidTokenException: - return False - - -def verify_robot(username, token): - """ Returns True if the given robot username and token match an existing robot - account. - """ - try: - return bool(model.user.verify_robot(username, token)) - except model.InvalidRobotException: - return False - - -def change_user_password(user, new_password): - """ Changes the password associated with the given user. """ - model.user.change_password(user, new_password) - - -def get_repository(namespace_name, repo_name): - """ Returns the repository with the given name under the given namespace or None if none. """ - repo = model.repository.get_repository(namespace_name, repo_name) - if repo is None: - return None - - return repository_for_repo(repo) - - -def create_repository(namespace_name, repo_name, user=None): - """ Creates a new repository under the given namespace with the given name, for the given user. - """ - model.repository.create_repository(namespace_name, repo_name, user) - - -def repository_is_public(namespace_name, repo_name): - """ Returns whether the repository with the given name under the given namespace is public. - If no matching repository was found, returns False. - """ - return model.repository.repository_is_public(namespace_name, repo_name) - - -def validate_oauth_token(token): - """ Returns whether the given OAuth token validates. """ - return bool(model.oauth.validate_access_token(token)) - - -def get_sorted_matching_repositories(search_term, only_public, can_read, limit): - """ Returns a sorted list of repositories matching the given search term. can_read is a callback - that will be invoked for each repository found, to filter results to only those visible to - the current user (if any). - """ - repos = model.repository.get_sorted_matching_repositories(search_term, only_public, can_read, - limit=limit) - - return [repository_for_repo(repo) for repo in repos] + @classmethod + def get_sorted_matching_repositories(cls, search_term, only_public, can_read, limit): + repos = model.repository.get_sorted_matching_repositories(search_term, only_public, can_read, + limit=limit) + return [_repository_for_repo(repo) for repo in repos] diff --git a/data/interfaces/v2.py b/data/interfaces/v2.py index ff3e151dd..5ef48798a 100644 --- a/data/interfaces/v2.py +++ b/data/interfaces/v2.py @@ -1,95 +1,74 @@ +from collections import namedtuple + +from namedlist import namedlist from peewee import IntegrityError from data import model, database from data.model import DataModelException -from image import Blob, BlobUpload, ManifestJSON, RepositoryReference, Tag from image.docker.v1 import DockerV1Metadata -from data.interfaces.common import repository_for_repo + _MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws" -def create_repository(namespace_name, repo_name, creating_user=None): - """ Creates a new repository under the specified namespace with the given name. The user supplied - is the user creating the repository, if any. + +class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])): """ - return model.repository.create_repository(namespace_name, repo_name, creating_user) - - -def repository_is_public(namespace_name, repo_name): - """ Returns true if the repository with the given name under the given namespace has public - visibility. + ManifestJSON represents a Manifest of any format. """ - return model.repository.repository_is_public(namespace_name, repo_name) - -def get_repository(namespace_name, repo_name): - """ Returns a repository tuple for the repository with the given name under the given namespace. - Returns None if no such repository was found. +class Tag(namedtuple('Tag', ['name', 'repository'])): """ - repo = model.repository.get_repository(namespace_name, repo_name) - if repo is None: - return None - - return repository_for_repo(repo) - - -def has_active_tag(namespace_name, repo_name, tag_name): - """ Returns whether there is an active tag for the tag with the given name under the matching - repository, if any, or None if none. + Tag represents a user-facing alias for referencing a set of Manifests. """ - try: - model.tag.get_active_tag(namespace_name, repo_name, tag_name) - return True - except database.RepositoryTag.DoesNotExist: - return False -def get_manifest_by_tag(namespace_name, repo_name, tag_name): - """ Returns the current manifest for the tag with the given name under the matching repository, - if any, or None if none. +class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count', + 'chunk_count', 'sha_state', 'location_name', + 'storage_metadata', 'piece_sha_state', 'piece_hashes', + 'repo_namespace_name', 'repo_name'])): """ - try: - manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name) - return ManifestJSON(digest=manifest.digest, json=manifest.json_data, media_type=_MEDIA_TYPE) - except model.InvalidManifestException: - return None - - -def get_manifest_by_digest(namespace_name, repo_name, digest): - """ Returns the manifest matching the given digest under the matching repository, if any, - or None if none. + BlobUpload represents the current state of an Blob being uploaded. """ - try: - manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest) - return ManifestJSON(digest=digest, json=manifest.json_data, media_type=_MEDIA_TYPE) - except model.InvalidManifestException: - return None -def delete_manifest_by_digest(namespace_name, repo_name, digest): - """ Deletes the manifest with the associated digest (if any) and returns all removed tags - that pointed to that manifest. If the manifest was not found, returns an empty list. +class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])): + """ + Blob represents an opaque binary blob saved to the storage system. """ - tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest) - def _tag_view(tag): - return Tag( - name=tag.name, - repository=RepositoryReference( - id=tag.repository_id, - name=repo_name, - namespace_name=namespace_name, - ) - ) - return [_tag_view(tag) for tag in tags] +class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])): + """ + RepositoryReference represents a reference to a Repository, without its full metadata. + """ + + +class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', + 'is_public'])): + """ + Repository represents a namespaced collection of tags. + """ + + +def _repository_for_repo(repo): + """ + Returns a Repository object representing the repo data model instance given. + """ + return Repository( + id=repo.id, + name=repo.name, + namespace_name=repo.namespace_user.username, + description=repo.description, + is_public=model.repository.is_repository_public(repo) + ) def _docker_v1_metadata(namespace_name, repo_name, repo_image): - """ Returns a DockerV1Metadata object for the given image under the repository with the given - namespace and name. Note that the namespace and name are passed here as an optimization, - and are *not checked* against the image. + """ + Returns a DockerV1Metadata object for the given image under the repository with the given + namespace and name. Note that the namespace and name are passed here as an optimization, and are + *not checked* against the image. """ return DockerV1Metadata( namespace_name=namespace_name, @@ -98,264 +77,474 @@ def _docker_v1_metadata(namespace_name, repo_name, repo_image): checksum=repo_image.v1_checksum, content_checksum=repo_image.storage.content_checksum, compat_json=repo_image.v1_json_metadata, - created=repo_image.created, comment=repo_image.comment, command=repo_image.command, - parent_image_id=None, # TODO: make sure this isn't needed anywhere, as it is expensive to lookup + # TODO: make sure this isn't needed anywhere, as it is expensive to lookup + parent_image_id=None, ) -def get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name): - """ Returns the Docker V1 metadata associated with the tag with the given name under the - matching repository, if any. If none, returns None. +class DockerRegistryV2DataInterface(object): """ - try: - repo_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name, include_storage=True) - return _docker_v1_metadata(namespace_name, repo_name, repo_image) - except DataModelException: - return None - - - -def get_docker_v1_metadata_by_image_id(namespace_name, repo_name, docker_image_ids): - """ Returns a map of Docker V1 metadata for each given image ID, matched under the repository - with the given namespace and name. Returns an empty map if the matching repository was not - found. + Interface that represents all data store interactions required by a Docker Registry v1. """ - repo = model.repository.get_repository(namespace_name, repo_name) - if repo is None: - return {} - images_query = model.image.lookup_repository_images(repo, docker_image_ids) - return {image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image) - for image in images_query} + @classmethod + def create_repository(cls, namespace_name, repo_name, creating_user=None): + """ + Creates a new repository under the specified namespace with the given name. The user supplied is + the user creating the repository, if any. + """ + raise NotImplementedError() + + @classmethod + def repository_is_public(cls, namespace_name, repo_name): + """ + Returns true if the repository with the given name under the given namespace has public + visibility. + """ + raise NotImplementedError() + + @classmethod + def get_repository(cls, namespace_name, repo_name): + """ + Returns a repository tuple for the repository with the given name under the given namespace. + Returns None if no such repository was found. + """ + raise NotImplementedError() + + @classmethod + def has_active_tag(cls, namespace_name, repo_name, tag_name): + """ + Returns whether there is an active tag for the tag with the given name under the matching + repository, if any, or none if none. + """ + raise NotImplementedError() + + @classmethod + def get_manifest_by_tag(cls, namespace_name, repo_name, tag_name): + """ + Returns the current manifest for the tag with the given name under the matching repository, if + any, or None if none. + """ + raise NotImplementedError() + + @classmethod + def get_manifest_by_digest(cls, namespace_name, repo_name, digest): + """ + Returns the manifest matching the given digest under the matching repository, if any, or None if + none. + """ + raise NotImplementedError() + + @classmethod + def delete_manifest_by_digest(cls, namespace_name, repo_name, digest): + """ + Deletes the manifest with the associated digest (if any) and returns all removed tags that + pointed to that manifest. If the manifest was not found, returns an empty list. + """ + raise NotImplementedError() + + @classmethod + def get_docker_v1_metadata_by_tag(cls, namespace_name, repo_name, tag_name): + """ + Returns the Docker V1 metadata associated with the tag with the given name under the matching + repository, if any. If none, returns None. + """ + raise NotImplementedError() + + @classmethod + def get_docker_v1_metadata_by_image_id(cls, namespace_name, repo_name, docker_image_ids): + """ + Returns a map of Docker V1 metadata for each given image ID, matched under the repository with + the given namespace and name. Returns an empty map if the matching repository was not found. + """ + raise NotImplementedError() + + @classmethod + def get_parents_docker_v1_metadata(cls, namespace_name, repo_name, docker_image_id): + """ + Returns an ordered list containing the Docker V1 metadata for each parent of the image with the + given docker ID under the matching repository. Returns an empty list if the image was not found. + """ + raise NotImplementedError() + + @classmethod + def create_manifest_and_update_tag(cls, namespace_name, repo_name, tag_name, manifest_digest, + manifest_bytes): + """ + Creates a new manifest with the given digest and byte data, and assigns the tag with the given + name under the matching repository to it. + """ + raise NotImplementedError() + + @classmethod + def synthesize_v1_image(cls, repository, storage, image_id, created, comment, command, + compat_json, parent_image_id): + """ + Synthesizes a V1 image under the specified repository, pointing to the given storage and returns + the V1 metadata for the synthesized image. + """ + raise NotImplementedError() + + @classmethod + def save_manifest(cls, namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest, + manifest_bytes): + """ + Saves a manifest pointing to the given leaf image, with the given manifest, under the matching + repository as a tag with the given name. + """ + raise NotImplementedError() + + @classmethod + def repository_tags(cls, namespace_name, repo_name, limit, offset): + """ + Returns the active tags under the repository with the given name and namespace. + """ + raise NotImplementedError() + + @classmethod + def get_visible_repositories(cls, username, limit, offset): + """ + Returns the repositories visible to the user with the given username, if any. + """ + raise NotImplementedError() + + @classmethod + def create_blob_upload(cls, namespace_name, repo_name, upload_uuid, location_name, + storage_metadata): + """ + Creates a blob upload under the matching repository with the given UUID and metadata. + Returns whether the matching repository exists. + """ + raise NotImplementedError() + + @classmethod + def blob_upload_by_uuid(cls, namespace_name, repo_name, upload_uuid): + """ + Searches for a blob upload with the given UUID under the given repository and returns it or None + if none. + """ + raise NotImplementedError() + + @classmethod + def update_blob_upload(cls, blob_upload): + """ + Saves any changes to the blob upload object given to the backing data store. + Fields that can change: + - uncompressed_byte_count + - piece_hashes + - piece_sha_state + - storage_metadata + - byte_count + - chunk_count + - sha_state + """ + raise NotImplementedError() + + @classmethod + def delete_blob_upload(cls, namespace_name, repo_name, uuid): + """ + Deletes the blob upload with the given uuid under the matching repository. If none, does + nothing. + """ + raise NotImplementedError() + + @classmethod + def create_blob_and_temp_tag(cls, namespace_name, repo_name, blob_digest, blob_upload, + expiration_sec): + """ + Creates a blob and links a temporary tag with the specified expiration to it under the matching + repository. + """ + raise NotImplementedError() + + @classmethod + def lookup_blobs_by_digest(cls, namespace_name, repo_name, digests): + """ + Returns all the blobs with matching digests found under the matching repository. If the + repository doesn't exist, returns {}. + """ + raise NotImplementedError() + + @classmethod + def get_blob_by_digest(cls, namespace_name, repo_name, digest): + """ + Returns the blob with the given digest under the matching repository or None if none. + """ + raise NotImplementedError() + + @classmethod + def save_bittorrent_pieces(cls, blob, piece_size, piece_bytes): + """ + Saves the BitTorrent piece hashes for the given blob. + """ + raise NotImplementedError() + + @classmethod + def get_blob_path(cls, blob): + """ + Once everything is moved over, this could be in util.registry and not even touch the database. + """ + raise NotImplementedError() -def get_parents_docker_v1_metadata(namespace_name, repo_name, docker_image_id): - """ Returns an ordered list containing the Docker V1 metadata for each parent of the image - with the given docker ID under the matching repository. Returns an empty list if the image - was not found. +class PreOCIModel(DockerRegistryV2DataInterface): """ - repo_image = model.image.get_repo_image(namespace_name, repo_name, docker_image_id) - if repo_image is None: - return [] - - parents = model.image.get_parent_images(namespace_name, repo_name, repo_image) - return [_docker_v1_metadata(namespace_name, repo_name, image) for image in parents] - - -def create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest_digest, - manifest_bytes): - """ Creates a new manifest with the given digest and byte data, and assigns the tag with the - given name under the matching repository to it. + PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema + before it was changed to support the OCI specification. """ - try: - model.tag.associate_generated_tag_manifest(namespace_name, repo_name, tag_name, - manifest_digest, manifest_bytes) - except IntegrityError: - # It's already there! - pass + @classmethod + def create_repository(cls, namespace_name, repo_name, creating_user=None): + return model.repository.create_repository(namespace_name, repo_name, creating_user) + @classmethod + def repository_is_public(cls, namespace_name, repo_name): + return model.repository.repository_is_public(namespace_name, repo_name) -def synthesize_v1_image(repository, storage, image_id, created, comment, command, compat_json, - parent_image_id): - """ Synthesizes a V1 image under the specified repository, pointing to the given storage - and returns the V1 metadata for the synthesized image. - """ - repo = model.repository.get_repository(repository.namespace_name, repository.name) - if repo is None: - raise DataModelException('Unknown repository: %s/%s' % (repository.namespace_name, - repository.name)) + @classmethod + def get_repository(cls, namespace_name, repo_name): + repo = model.repository.get_repository(namespace_name, repo_name) + if repo is None: + return None + return _repository_for_repo(repo) - parent_image = None - if parent_image_id is not None: - parent_image = model.image.get_image(repo, parent_image_id) - if parent_image is None: - raise DataModelException('Unknown parent image: %s' % parent_image_id) + @classmethod + def has_active_tag(cls, namespace_name, repo_name, tag_name): + try: + model.tag.get_active_tag(namespace_name, repo_name, tag_name) + return True + except database.RepositoryTag.DoesNotExist: + return False - storage_obj = model.storage.get_storage_by_uuid(storage.uuid) - if storage_obj is None: - raise DataModelException('Unknown storage: %s' % storage.uuid) + @classmethod + def get_manifest_by_tag(cls, namespace_name, repo_name, tag_name): + try: + manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name) + return ManifestJSON(digest=manifest.digest, json=manifest.json_data, media_type=_MEDIA_TYPE) + except model.InvalidManifestException: + return None - repo_image = model.image.synthesize_v1_image(repo, storage_obj, image_id, created, comment, - command, compat_json, parent_image) - return _docker_v1_metadata(repo.namespace_user.username, repo.name, repo_image) + @classmethod + def get_manifest_by_digest(cls, namespace_name, repo_name, digest): + try: + manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest) + return ManifestJSON(digest=digest, json=manifest.json_data, media_type=_MEDIA_TYPE) + except model.InvalidManifestException: + return None - -def save_manifest(namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest, - manifest_bytes): - """ Saves a manifest pointing to the given leaf image, with the given manifest, under the matching - repository as a tag with the given name. - """ - model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_docker_id, - manifest_digest, manifest_bytes) - - -def repository_tags(namespace_name, repo_name, limit, offset): - """ Returns the active tags under the repository with the given name and namespace. """ - tags_query = model.tag.list_repository_tags(namespace_name, repo_name) - tags_query = tags_query.limit(limit).offset(offset) - - def _tag_view(tag): - return Tag( - name=tag.name, - repository=RepositoryReference( - id=tag.repository_id, - name=repo_name, - namespace_name=namespace_name, + @classmethod + def delete_manifest_by_digest(cls, namespace_name, repo_name, digest): + def _tag_view(tag): + return Tag( + name=tag.name, + repository=RepositoryReference( + id=tag.repository_id, + name=repo_name, + namespace_name=namespace_name, + ) ) + + tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest) + return [_tag_view(tag) for tag in tags] + + @classmethod + def get_docker_v1_metadata_by_tag(cls, namespace_name, repo_name, tag_name): + try: + repo_img = model.tag.get_tag_image(namespace_name, repo_name, tag_name, include_storage=True) + return _docker_v1_metadata(namespace_name, repo_name, repo_img) + except DataModelException: + return None + + @classmethod + def get_docker_v1_metadata_by_image_id(cls, namespace_name, repo_name, docker_image_ids): + repo = model.repository.get_repository(namespace_name, repo_name) + if repo is None: + return {} + + images_query = model.image.lookup_repository_images(repo, docker_image_ids) + return {image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image) + for image in images_query} + + @classmethod + def get_parents_docker_v1_metadata(cls, namespace_name, repo_name, docker_image_id): + repo_image = model.image.get_repo_image(namespace_name, repo_name, docker_image_id) + if repo_image is None: + return [] + + parents = model.image.get_parent_images(namespace_name, repo_name, repo_image) + return [_docker_v1_metadata(namespace_name, repo_name, image) for image in parents] + + @classmethod + def create_manifest_and_update_tag(cls, namespace_name, repo_name, tag_name, manifest_digest, + manifest_bytes): + try: + model.tag.associate_generated_tag_manifest(namespace_name, repo_name, tag_name, + manifest_digest, manifest_bytes) + except IntegrityError: + # It's already there! + pass + + @classmethod + def synthesize_v1_image(cls, repository, storage, image_id, created, comment, command, + compat_json, parent_image_id): + repo = model.repository.get_repository(repository.namespace_name, repository.name) + if repo is None: + raise DataModelException('Unknown repository: %s/%s' % (repository.namespace_name, + repository.name)) + + parent_image = None + if parent_image_id is not None: + parent_image = model.image.get_image(repo, parent_image_id) + if parent_image is None: + raise DataModelException('Unknown parent image: %s' % parent_image_id) + + storage_obj = model.storage.get_storage_by_uuid(storage.uuid) + if storage_obj is None: + raise DataModelException('Unknown storage: %s' % storage.uuid) + + repo_image = model.image.synthesize_v1_image(repo, storage_obj, image_id, created, comment, + command, compat_json, parent_image) + return _docker_v1_metadata(repo.namespace_user.username, repo.name, repo_image) + + @classmethod + def save_manifest(cls, namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest, + manifest_bytes): + model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_docker_id, + manifest_digest, manifest_bytes) + + @classmethod + def repository_tags(cls, namespace_name, repo_name, limit, offset): + def _tag_view(tag): + return Tag( + name=tag.name, + repository=RepositoryReference( + id=tag.repository_id, + name=repo_name, + namespace_name=namespace_name, + ) + ) + + tags_query = model.tag.list_repository_tags(namespace_name, repo_name) + tags_query = tags_query.limit(limit).offset(offset) + return [_tag_view(tag) for tag in tags_query] + + @classmethod + def get_visible_repositories(cls, username, limit, offset): + query = model.repository.get_visible_repositories(username, include_public=(username is None)) + query = query.limit(limit).offset(offset) + return [_repository_for_repo(repo) for repo in query] + + @classmethod + def create_blob_upload(cls, namespace_name, repo_name, upload_uuid, location_name, + storage_metadata): + try: + model.blob.initiate_upload(namespace_name, repo_name, upload_uuid, location_name, + storage_metadata) + return True + except database.Repository.DoesNotExist: + return False + + @classmethod + def blob_upload_by_uuid(cls, namespace_name, repo_name, upload_uuid): + try: + found = model.blob.get_blob_upload(namespace_name, repo_name, upload_uuid) + except model.InvalidBlobUpload: + return None + + return BlobUpload( + repo_namespace_name=namespace_name, + repo_name=repo_name, + uuid=upload_uuid, + byte_count=found.byte_count, + uncompressed_byte_count=found.uncompressed_byte_count, + chunk_count=found.chunk_count, + sha_state=found.sha_state, + piece_sha_state=found.piece_sha_state, + piece_hashes=found.piece_hashes, + location_name=found.location.name, + storage_metadata=found.storage_metadata, ) - return [_tag_view(tag) for tag in tags_query] + @classmethod + def update_blob_upload(cls, blob_upload): + # Lookup the blob upload object. + try: + blob_upload_record = model.blob.get_blob_upload(blob_upload.repo_namespace_name, + blob_upload.repo_name, blob_upload.uuid) + except model.InvalidBlobUpload: + return + blob_upload_record.uncompressed_byte_count = blob_upload.uncompressed_byte_count + blob_upload_record.piece_hashes = blob_upload.piece_hashes + blob_upload_record.piece_sha_state = blob_upload.piece_sha_state + blob_upload_record.storage_metadata = blob_upload.storage_metadata + blob_upload_record.byte_count = blob_upload.byte_count + blob_upload_record.chunk_count = blob_upload.chunk_count + blob_upload_record.sha_state = blob_upload.sha_state + blob_upload_record.save() -def get_visible_repositories(username, limit, offset): - """ Returns the repositories visible to the user with the given username, if any. """ - query = model.repository.get_visible_repositories(username, include_public=(username is None)) - query = query.limit(limit).offset(offset) - return [repository_for_repo(repo) for repo in query] + @classmethod + def delete_blob_upload(cls, namespace_name, repo_name, uuid): + try: + found = model.blob.get_blob_upload(namespace_name, repo_name, uuid) + found.delete_instance() + except model.InvalidBlobUpload: + return - -def create_blob_upload(namespace_name, repo_name, upload_uuid, location_name, storage_metadata): - """ Creates a blob upload under the matching repository with the given UUID and metadata. - Returns whether the matching repository exists. - """ - try: - model.blob.initiate_upload(namespace_name, repo_name, upload_uuid, location_name, - storage_metadata) - return True - except database.Repository.DoesNotExist: - return False - - -def blob_upload_by_uuid(namespace_name, repo_name, upload_uuid): - """ Searches for a blob upload with the given UUID under the given repository and returns it - or None if none. - """ - try: - found = model.blob.get_blob_upload(namespace_name, repo_name, upload_uuid) - except model.InvalidBlobUpload: - return None - - return BlobUpload( - repo_namespace_name=namespace_name, - repo_name=repo_name, - uuid=upload_uuid, - byte_count=found.byte_count, - uncompressed_byte_count=found.uncompressed_byte_count, - chunk_count=found.chunk_count, - sha_state=found.sha_state, - piece_sha_state=found.piece_sha_state, - piece_hashes=found.piece_hashes, - location_name=found.location.name, - storage_metadata=found.storage_metadata, - ) - - -def update_blob_upload(blob_upload): - """ Saves any changes to the blob upload object given to the backing data store. - - Fields that can change: - - uncompressed_byte_count - - piece_hashes - - piece_sha_state - - storage_metadata - - byte_count - - chunk_count - - sha_state - """ - # Lookup the blob upload object. - try: - blob_upload_record = model.blob.get_blob_upload(blob_upload.repo_namespace_name, - blob_upload.repo_name, blob_upload.uuid) - except model.InvalidBlobUpload: - return - - blob_upload_record.uncompressed_byte_count = blob_upload.uncompressed_byte_count - blob_upload_record.piece_hashes = blob_upload.piece_hashes - blob_upload_record.piece_sha_state = blob_upload.piece_sha_state - blob_upload_record.storage_metadata = blob_upload.storage_metadata - blob_upload_record.byte_count = blob_upload.byte_count - blob_upload_record.chunk_count = blob_upload.chunk_count - blob_upload_record.sha_state = blob_upload.sha_state - blob_upload_record.save() - - -def delete_blob_upload(namespace_name, repo_name, uuid): - """ Deletes the blob upload with the given uuid under the matching repository. If none, does - nothing. - """ - try: - found = model.blob.get_blob_upload(namespace_name, repo_name, uuid) - except model.InvalidBlobUpload: - return - - found.delete_instance() - - -def create_blob_and_temp_tag(namespace_name, repo_name, blob_digest, blob_upload, expiration_sec): - """ Crates a blob and links a temporary tag with the specified expiration to it under the - matching repository. - """ - location_obj = model.storage.get_image_location_for_name(blob_upload.location_name) - blob_record = model.blob.store_blob_record_and_temp_link(namespace_name, repo_name, - blob_digest, - location_obj.id, - blob_upload.byte_count, - expiration_sec, - blob_upload.uncompressed_byte_count) - return Blob( - uuid=blob_record.uuid, - digest=blob_digest, - size=blob_upload.byte_count, - locations=[blob_upload.location_name], - ) - - -def lookup_blobs_by_digest(namespace_name, repo_name, digests): - """ Returns all the blobs with matching digests found under the matching repository. If the - repository doesn't exist, returns {}. - """ - repo = model.repository.get_repository(namespace_name, repo_name) - if repo is None: - return {} - - def _blob_view(blob_record): + @classmethod + def create_blob_and_temp_tag(cls, namespace_name, repo_name, blob_digest, blob_upload, + expiration_sec): + location_obj = model.storage.get_image_location_for_name(blob_upload.location_name) + blob_record = model.blob.store_blob_record_and_temp_link(namespace_name, repo_name, + blob_digest, + location_obj.id, + blob_upload.byte_count, + expiration_sec, + blob_upload.uncompressed_byte_count) return Blob( uuid=blob_record.uuid, - digest=blob_record.content_checksum, - size=blob_record.image_size, - locations=None, # Note: Locations is None in this case. + digest=blob_digest, + size=blob_upload.byte_count, + locations=[blob_upload.location_name], ) - query = model.storage.lookup_repo_storages_by_content_checksum(repo, digests) - return {storage.content_checksum: _blob_view(storage) for storage in query} + @classmethod + def lookup_blobs_by_digest(cls, namespace_name, repo_name, digests): + def _blob_view(blob_record): + return Blob( + uuid=blob_record.uuid, + digest=blob_record.content_checksum, + size=blob_record.image_size, + locations=None, # Note: Locations is None in this case. + ) + repo = model.repository.get_repository(namespace_name, repo_name) + if repo is None: + return {} + query = model.storage.lookup_repo_storages_by_content_checksum(repo, digests) + return {storage.content_checksum: _blob_view(storage) for storage in query} -def get_blob_by_digest(namespace_name, repo_name, digest): - """ Returns the blob with the given digest under the matching repository or None if none. """ - try: - blob_record = model.blob.get_repo_blob_by_digest(namespace_name, repo_name, digest) - return Blob( - uuid=blob_record.uuid, - digest=digest, - size=blob_record.image_size, - locations=blob_record.locations, - ) - except model.BlobDoesNotExist: - return None + @classmethod + def get_blob_by_digest(cls, namespace_name, repo_name, digest): + try: + blob_record = model.blob.get_repo_blob_by_digest(namespace_name, repo_name, digest) + return Blob( + uuid=blob_record.uuid, + digest=digest, + size=blob_record.image_size, + locations=blob_record.locations, + ) + except model.BlobDoesNotExist: + return None + @classmethod + def save_bittorrent_pieces(cls, blob, piece_size, piece_bytes): + blob_record = model.storage.get_storage_by_uuid(blob.uuid) + model.storage.save_torrent_info(blob_record, piece_size, piece_bytes) -def save_bittorrent_pieces(blob, piece_size, piece_bytes): - """ Saves the BitTorrent piece hashes for the given blob. """ - blob_record = model.storage.get_storage_by_uuid(blob.uuid) - model.storage.save_torrent_info(blob_record, piece_size, piece_bytes) - - -def get_blob_path(blob): - # Once everything is moved over, this could be in util.registry and not even - # touch the database. - blob_record = model.storage.get_storage_by_uuid(blob.uuid) - return model.storage.get_layer_path(blob_record) - + @classmethod + def get_blob_path(cls, blob): + blob_record = model.storage.get_storage_by_uuid(blob.uuid) + return model.storage.get_layer_path(blob_record) diff --git a/endpoints/v1/index.py b/endpoints/v1/index.py index 250731a11..c90ffc9d5 100644 --- a/endpoints/v1/index.py +++ b/endpoints/v1/index.py @@ -6,9 +6,8 @@ from functools import wraps from flask import request, make_response, jsonify, session -from data.interfaces import v1 +from data.interfaces.v1 import PreOCIModel as model from app import authentication, userevents, metric_queue -from app import authentication, userevents from auth.auth import process_auth, generate_signed_token from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission, @@ -86,17 +85,17 @@ def create_user(): success = make_response('"Username or email already exists"', 400) if username == '$token': - if v1.load_token(password): + if model.load_token(password): return success abort(400, 'Invalid access token.', issue='invalid-access-token') elif username == '$oauthtoken': - if v1.validate_oauth_token(password): + if model.validate_oauth_token(password): return success abort(400, 'Invalid oauth access token.', issue='invalid-oauth-access-token') elif '+' in username: - if v1.verify_robot(username, password): + if model.verify_robot(username, password): return success abort(400, 'Invalid robot account or password.', issue='robot-login-failure') @@ -147,7 +146,7 @@ def update_user(username): if 'password' in update_request: logger.debug('Updating user password') - v1.change_user_password(get_authenticated_user(), update_request['password']) + model.change_user_password(get_authenticated_user(), update_request['password']) return jsonify({ 'username': get_authenticated_user().username, @@ -167,7 +166,7 @@ def create_repository(namespace_name, repo_name): abort(400, message='Invalid repository name. Repository names cannot contain slashes.') logger.debug('Looking up repository %s/%s', namespace_name, repo_name) - repo = v1.get_repository(namespace_name, repo_name) + repo = model.get_repository(namespace_name, repo_name) logger.debug('Found repository %s/%s', namespace_name, repo_name) if not repo and get_authenticated_user() is None: @@ -195,7 +194,7 @@ def create_repository(namespace_name, repo_name): logger.debug('Creating repository %s/%s with owner: %s', namespace_name, repo_name, get_authenticated_user().username) - v1.create_repository(namespace_name, repo_name, get_authenticated_user()) + model.create_repository(namespace_name, repo_name, get_authenticated_user()) if get_authenticated_user(): user_event_data = { @@ -220,7 +219,7 @@ def update_images(namespace_name, repo_name): if permission.can(): logger.debug('Looking up repository') - repo = v1.get_repository(namespace_name, repo_name) + repo = model.get_repository(namespace_name, repo_name) if not repo: # Make sure the repo actually exists. abort(404, message='Unknown repository', issue='unknown-repo') @@ -250,10 +249,10 @@ def get_repository_images(namespace_name, repo_name): permission = ReadRepositoryPermission(namespace_name, repo_name) # TODO invalidate token? - if permission.can() or v1.repository_is_public(namespace_name, repo_name): + if permission.can() or model.repository_is_public(namespace_name, repo_name): # We can't rely on permissions to tell us if a repo exists anymore logger.debug('Looking up repository') - repo = v1.get_repository(namespace_name, repo_name) + repo = model.get_repository(namespace_name, repo_name) if not repo: abort(404, message='Unknown repository', issue='unknown-repo') @@ -319,7 +318,7 @@ def _conduct_repo_search(username, query, results): return ReadRepositoryPermission(repo.namespace_name, repo.name).can() only_public = username is None - matching_repos = v1.get_sorted_matching_repositories(query, only_public, can_read, limit=5) + matching_repos = model.get_sorted_matching_repositories(query, only_public, can_read, limit=5) for repo in matching_repos: results.append({ diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py index 1a4ca4fc3..1f169db0a 100644 --- a/endpoints/v1/registry.py +++ b/endpoints/v1/registry.py @@ -14,7 +14,7 @@ from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission) from auth.registry_jwt_auth import get_granted_username from data import model, database -from data.interfaces import v1 +from data.interfaces.v1 import PreOCIModel as model from digest import checksums from endpoints.v1 import v1_bp from endpoints.decorators import anon_protect @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) def _finish_image(namespace, repository, image_id): # Checksum is ok, we remove the marker - blob_ref = v1.update_image_uploading(namespace, repository, image_id, False) + blob_ref = model.update_image_uploading(namespace, repository, image_id, False) # Send a job to the work queue to replicate the image layer. queue_storage_replication(namespace, blob_ref) @@ -41,7 +41,7 @@ def require_completion(f): @wraps(f) def wrapper(namespace, repository, *args, **kwargs): image_id = kwargs['image_id'] - if v1.is_image_uploading(namespace, repository, image_id): + if model.is_image_uploading(namespace, repository, image_id): abort(400, 'Image %(image_id)s is being uploaded, retry later', issue='upload-in-progress', image_id=image_id) return f(namespace, repository, *args, **kwargs) @@ -82,9 +82,9 @@ def head_image_layer(namespace, repository, image_id, headers): permission = ReadRepositoryPermission(namespace, repository) logger.debug('Checking repo permissions') - if permission.can() or model.repository.repository_is_public(namespace, repository): + if permission.can() or model.repository_is_public(namespace, repository): logger.debug('Looking up placement locations') - locations = v1.placement_locations_docker_v1(namespace, repository, image_id) + locations = model.placement_locations_docker_v1(namespace, repository, image_id) if locations is None: logger.debug('Could not find any blob placement locations') abort(404, 'Image %(image_id)s not found', issue='unknown-image', @@ -115,11 +115,9 @@ def get_image_layer(namespace, repository, image_id, headers): permission = ReadRepositoryPermission(namespace, repository) logger.debug('Checking repo permissions') - if permission.can() or model.repository.repository_is_public(namespace, repository): + if permission.can() or model.repository_is_public(namespace, repository): logger.debug('Looking up placement locations and path') - locations, path = v1.placement_locations_and_path_docker_v1(namespace, - repository, - image_id) + locations, path = model.placement_locations_and_path_docker_v1(namespace, repository, image_id) if not locations or not path: abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) @@ -154,7 +152,7 @@ def put_image_layer(namespace, repository, image_id): abort(403) logger.debug('Retrieving image') - if v1.storage_exists(namespace, repository, image_id): + if model.storage_exists(namespace, repository, image_id): exact_abort(409, 'Image already exists') logger.debug('Storing layer data') @@ -184,7 +182,7 @@ def put_image_layer(namespace, repository, image_id): sr.add_handler(piece_hasher.update) # Add a handler which computes the checksum. - v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) + v1_metadata = model.docker_v1_metadata(namespace, repository, image_id) h, sum_hndlr = checksums.simple_checksum_handler(v1_metadata.compat_json) sr.add_handler(sum_hndlr) @@ -193,7 +191,7 @@ def put_image_layer(namespace, repository, image_id): sr.add_handler(content_sum_hndlr) # Stream write the data to storage. - locations, path = v1.placement_locations_and_path_docker_v1(namespace, repository, image_id) + locations, path = model.placement_locations_and_path_docker_v1(namespace, repository, image_id) with database.CloseForLongOperation(app.config): try: store.stream_write(locations, path, sr) @@ -202,11 +200,11 @@ def put_image_layer(namespace, repository, image_id): abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id) # Save the size of the image. - v1.update_image_sizes(namespace, repository, image_id, size_info.compressed_size, - size_info.uncompressed_size) + model.update_image_sizes(namespace, repository, image_id, size_info.compressed_size, + size_info.uncompressed_size) # Save the BitTorrent pieces. - v1.create_bittorrent_pieces(namespace, repository, image_id, piece_hasher.final_piece_hashes()) + model.create_bittorrent_pieces(namespace, repository, image_id, piece_hasher.final_piece_hashes()) # Append the computed checksum. csums = [] @@ -271,7 +269,7 @@ def put_image_checksum(namespace, repository, image_id): issue='missing-checksum-cookie', image_id=image_id) logger.debug('Looking up repo image') - v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) + v1_metadata = model.docker_v1_metadata(namespace, repository, image_id) if not v1_metadata: abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) @@ -280,7 +278,7 @@ def put_image_checksum(namespace, repository, image_id): abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) logger.debug('Marking image path') - if not v1.is_image_uploading(namespace, repository, image_id): + if not model.is_image_uploading(namespace, repository, image_id): abort(409, 'Cannot set checksum for image %(image_id)s', issue='image-write-error', image_id=image_id) @@ -291,7 +289,7 @@ def put_image_checksum(namespace, repository, image_id): if len(checksum_parts) != 2: abort(400, 'Invalid checksum format') - v1.store_docker_v1_checksums(namespace, repository, image_id, checksum, content_checksum) + model.store_docker_v1_checksums(namespace, repository, image_id, checksum, content_checksum) if checksum not in session.get('checksum', []): logger.debug('session checksums: %s', session.get('checksum', [])) @@ -315,16 +313,16 @@ def put_image_checksum(namespace, repository, image_id): def get_image_json(namespace, repository, image_id, headers): logger.debug('Checking repo permissions') permission = ReadRepositoryPermission(namespace, repository) - if not permission.can() and not model.repository.repository_is_public(namespace, repository): + if not permission.can() and not model.repository_is_public(namespace, repository): abort(403) logger.debug('Looking up repo image') - v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) + v1_metadata = model.docker_v1_metadata(namespace, repository, image_id) if v1_metadata is None: flask_abort(404) logger.debug('Looking up repo layer size') - size = v1.get_image_size(namespace, repository, image_id) + size = model.get_image_size(namespace, repository, image_id) if size is not None: # Note: X-Docker-Size is optional and we *can* end up with a NULL image_size, # so handle this case rather than failing. @@ -344,10 +342,10 @@ def get_image_json(namespace, repository, image_id, headers): def get_image_ancestry(namespace, repository, image_id, headers): logger.debug('Checking repo permissions') permission = ReadRepositoryPermission(namespace, repository) - if not permission.can() and not model.repository.repository_is_public(namespace, repository): + if not permission.can() and not model.repository_is_public(namespace, repository): abort(403) - ancestry_docker_ids = v1.image_ancestry(namespace, repository, image_id) + ancestry_docker_ids = model.image_ancestry(namespace, repository, image_id) if ancestry_docker_ids is None: abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) @@ -388,37 +386,39 @@ def put_image_json(namespace, repository, image_id): logger.debug('Looking up repo image') - if not v1.repository_exists(namespace, repository): + if not model.repository_exists(namespace, repository): abort(404, 'Repository does not exist: %(namespace)s/%(repository)s', issue='no-repo', namespace=namespace, repository=repository) - v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) + v1_metadata = model.docker_v1_metadata(namespace, repository, image_id) if v1_metadata is None: username = get_authenticated_user() and get_authenticated_user().username if not username: username = get_granted_username() - logger.debug('Image not found, creating or linking image with initiating user context: %s', username) - v1.create_or_link_image(username, namespace, repository, image_id, store.preferred_locations[0]) - v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) + logger.debug('Image not found, creating or linking image with initiating user context: %s', + username) + location_pref = store.preferred_locations[0] + model.create_or_link_image(username, namespace, repository, image_id, location_pref) + v1_metadata = model.docker_v1_metadata(namespace, repository, image_id) # Create a temporary tag to prevent this image from getting garbage collected while the push # is in progress. - v1.create_temp_hidden_tag(namespace, repository, image_id, - app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']) + model.create_temp_hidden_tag(namespace, repository, image_id, + app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']) parent_id = data.get('parent', None) if parent_id: logger.debug('Looking up parent image') - if v1.docker_v1_metadata(namespace, repository, parent_id) is None: + if model.docker_v1_metadata(namespace, repository, parent_id) is None: abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s', issue='invalid-request', image_id=image_id, parent_id=parent_id) logger.debug('Checking if image already exists') - if v1_metadata and not v1.is_image_uploading(namespace, repository, image_id): + if v1_metadata and not model.is_image_uploading(namespace, repository, image_id): exact_abort(409, 'Image already exists') - v1.update_image_uploading(namespace, repository, image_id, True) + model.update_image_uploading(namespace, repository, image_id, True) # If we reach that point, it means that this is a new image or a retry # on a failed push, save the metadata @@ -426,7 +426,7 @@ def put_image_json(namespace, repository, image_id): command = json.dumps(command_list) if command_list else None logger.debug('Setting image metadata') - v1.update_docker_v1_metadata(namespace, repository, image_id, data.get('created'), - data.get('comment'), command, uploaded_metadata, parent_id) + model.update_docker_v1_metadata(namespace, repository, image_id, data.get('created'), + data.get('comment'), command, uploaded_metadata, parent_id) return make_response('true', 200) diff --git a/endpoints/v1/tag.py b/endpoints/v1/tag.py index 822e55cb1..07902309d 100644 --- a/endpoints/v1/tag.py +++ b/endpoints/v1/tag.py @@ -9,7 +9,7 @@ from auth.auth import process_auth from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission) from data import model -from data.interfaces import v1 +from data.interfaces.v1 import PreOCIModel as model from endpoints.common import parse_repository_name from endpoints.decorators import anon_protect from endpoints.v1 import v1_bp @@ -26,8 +26,8 @@ logger = logging.getLogger(__name__) def get_tags(namespace_name, repo_name): permission = ReadRepositoryPermission(namespace_name, repo_name) - if permission.can() or model.repository.repository_is_public(namespace_name, repo_name): - tags = v1.list_tags(namespace_name, repo_name) + if permission.can() or model.repository_is_public(namespace_name, repo_name): + tags = model.list_tags(namespace_name, repo_name) tag_map = {tag.name: tag.image.docker_image_id for tag in tags} return jsonify(tag_map) @@ -41,8 +41,8 @@ def get_tags(namespace_name, repo_name): def get_tag(namespace_name, repo_name, tag): permission = ReadRepositoryPermission(namespace_name, repo_name) - if permission.can() or model.repository.repository_is_public(namespace_name, repo_name): - image_id = v1.find_image_id_by_tag(namespace_name, repo_name, tag) + if permission.can() or model.repository_is_public(namespace_name, repo_name): + image_id = model.find_image_id_by_tag(namespace_name, repo_name, tag) if image_id is None: abort(404) @@ -65,7 +65,7 @@ def put_tag(namespace_name, repo_name, tag): abort(400, TAG_ERROR) image_id = json.loads(request.data) - v1.create_or_update_tag(namespace_name, repo_name, image_id, tag) + model.create_or_update_tag(namespace_name, repo_name, image_id, tag) # Store the updated tag. if 'pushed_tags' not in session: @@ -86,9 +86,8 @@ def delete_tag(namespace_name, repo_name, tag): permission = ModifyRepositoryPermission(namespace_name, repo_name) if permission.can(): - v1.delete_tag(namespace_name, repo_name, tag) - track_and_log('delete_tag', model.repository.get_repository(namespace_name, repo_name), - tag=tag) + model.delete_tag(namespace_name, repo_name, tag) + track_and_log('delete_tag', model.get_repository(namespace_name, repo_name), tag=tag) return make_response('Deleted', 200) abort(403) diff --git a/endpoints/v2/blob.py b/endpoints/v2/blob.py index 504a7d83e..1e1a2bd30 100644 --- a/endpoints/v2/blob.py +++ b/endpoints/v2/blob.py @@ -8,7 +8,7 @@ import resumablehashlib from app import storage, app from auth.registry_jwt_auth import process_registry_jwt_auth from data import database -from data.interfaces import v2 +from data.interfaces.v2 import PreOCIModel as model from digest import digest_tools from endpoints.common import parse_repository_name from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream @@ -42,7 +42,7 @@ class _InvalidRangeHeader(Exception): @cache_control(max_age=31436000) def check_blob_exists(namespace_name, repo_name, digest): # Find the blob. - blob = v2.get_blob_by_digest(namespace_name, repo_name, digest) + blob = model.get_blob_by_digest(namespace_name, repo_name, digest) if blob is None: raise BlobUnknown() @@ -69,7 +69,7 @@ def check_blob_exists(namespace_name, repo_name, digest): @cache_control(max_age=31536000) def download_blob(namespace_name, repo_name, digest): # Find the blob. - blob = v2.get_blob_by_digest(namespace_name, repo_name, digest) + blob = model.get_blob_by_digest(namespace_name, repo_name, digest) if blob is None: raise BlobUnknown() @@ -81,7 +81,7 @@ def download_blob(namespace_name, repo_name, digest): headers['Accept-Ranges'] = 'bytes' # Find the storage path for the blob. - path = v2.get_blob_path(blob) + path = model.get_blob_path(blob) # Short-circuit by redirecting if the storage supports it. logger.debug('Looking up the direct download URL for path: %s', path) @@ -115,8 +115,8 @@ def start_blob_upload(namespace_name, repo_name): # Begin the blob upload process in the database and storage. location_name = storage.preferred_locations[0] new_upload_uuid, upload_metadata = storage.initiate_chunked_upload(location_name) - repository_exists = v2.create_blob_upload(namespace_name, repo_name, new_upload_uuid, - location_name, upload_metadata) + repository_exists = model.create_blob_upload(namespace_name, repo_name, new_upload_uuid, + location_name, upload_metadata) if not repository_exists: raise NameUnknown() @@ -135,7 +135,7 @@ def start_blob_upload(namespace_name, repo_name): # The user plans to send us the entire body right now. # Find the upload. - blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, new_upload_uuid) + blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, new_upload_uuid) if blob_upload is None: raise BlobUploadUnknown() @@ -146,7 +146,7 @@ def start_blob_upload(namespace_name, repo_name): _abort_range_not_satisfiable(blob_upload.byte_count, new_upload_uuid) # Save the upload state to the database. - v2.update_blob_upload(updated_blob_upload) + model.update_blob_upload(updated_blob_upload) # Finalize the upload process in the database and storage. _finish_upload(namespace_name, repo_name, updated_blob_upload, digest) @@ -168,7 +168,7 @@ def start_blob_upload(namespace_name, repo_name): @require_repo_write @anon_protect def fetch_existing_upload(namespace_name, repo_name, upload_uuid): - blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) + blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) if blob_upload is None: raise BlobUploadUnknown() @@ -188,7 +188,7 @@ def fetch_existing_upload(namespace_name, repo_name, upload_uuid): @anon_protect def upload_chunk(namespace_name, repo_name, upload_uuid): # Find the upload. - blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) + blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) if blob_upload is None: raise BlobUploadUnknown() @@ -199,7 +199,7 @@ def upload_chunk(namespace_name, repo_name, upload_uuid): _abort_range_not_satisfiable(blob_upload.byte_count, upload_uuid) # Save the upload state to the database. - v2.update_blob_upload(updated_blob_upload) + model.update_blob_upload(updated_blob_upload) # Write the response to the client. return Response( @@ -224,7 +224,7 @@ def monolithic_upload_or_last_chunk(namespace_name, repo_name, upload_uuid): raise BlobUploadInvalid(detail={'reason': 'Missing digest arg on monolithic upload'}) # Find the upload. - blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) + blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) if blob_upload is None: raise BlobUploadUnknown() @@ -254,13 +254,13 @@ def monolithic_upload_or_last_chunk(namespace_name, repo_name, upload_uuid): @require_repo_write @anon_protect def cancel_upload(namespace_name, repo_name, upload_uuid): - blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) + blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) if blob_upload is None: raise BlobUploadUnknown() # We delete the record for the upload first, since if the partial upload in # storage fails to delete, it doesn't break anything. - v2.delete_blob_upload(namespace_name, repo_name, upload_uuid) + model.delete_blob_upload(namespace_name, repo_name, upload_uuid) storage.cancel_chunked_upload({blob_upload.location_name}, blob_upload.uuid, blob_upload.storage_metadata) @@ -471,7 +471,7 @@ def _finalize_blob_database(namespace_name, repo_name, blob_upload, digest, alre database's perspective. """ # Create the blob and temporarily tag it. - blob_storage = v2.create_blob_and_temp_tag( + blob_storage = model.create_blob_and_temp_tag( namespace_name, repo_name, digest, @@ -482,10 +482,10 @@ def _finalize_blob_database(namespace_name, repo_name, blob_upload, digest, alre # If it doesn't already exist, create the BitTorrent pieces for the blob. if blob_upload.piece_sha_state is not None and not already_existed: piece_bytes = blob_upload.piece_hashes + blob_upload.piece_sha_state.digest() - v2.save_bittorrent_pieces(blob_storage, app.config['BITTORRENT_PIECE_SIZE'], piece_bytes) + model.save_bittorrent_pieces(blob_storage, app.config['BITTORRENT_PIECE_SIZE'], piece_bytes) # Delete the blob upload. - v2.delete_blob_upload(namespace_name, repo_name, blob_upload.uuid) + model.delete_blob_upload(namespace_name, repo_name, blob_upload.uuid) def _finish_upload(namespace_name, repo_name, blob_upload, digest): diff --git a/endpoints/v2/catalog.py b/endpoints/v2/catalog.py index 9586c7909..4a145fe3f 100644 --- a/endpoints/v2/catalog.py +++ b/endpoints/v2/catalog.py @@ -3,7 +3,7 @@ from flask import jsonify from auth.registry_jwt_auth import process_registry_jwt_auth, get_granted_entity from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp, paginate -from data.interfaces import v2 +from data.interfaces.v2 import PreOCIModel as model @v2_bp.route('/_catalog', methods=['GET']) @process_registry_jwt_auth() @@ -15,7 +15,7 @@ def catalog_search(limit, offset, pagination_callback): if entity: username = entity.user.username - visible_repositories = v2.get_visible_repositories(username, limit+1, offset) + visible_repositories = model.get_visible_repositories(username, limit+1, offset) response = jsonify({ 'repositories': ['%s/%s' % (repo.namespace_name, repo.name) for repo in visible_repositories][0:limit], diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index 0fdd5cb51..9fdbe6ed1 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -8,8 +8,7 @@ import features from app import docker_v2_signing_key, app, metric_queue from auth.registry_jwt_auth import process_registry_jwt_auth -from data import model -from data.interfaces import v2 +from data.interfaces.v2 import PreOCIModel as model from digest import digest_tools from endpoints.common import parse_repository_name from endpoints.decorators import anon_protect @@ -24,6 +23,7 @@ from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES from util.registry.replication import queue_storage_replication from util.names import VALID_TAG_PATTERN + logger = logging.getLogger(__name__) @@ -37,9 +37,9 @@ MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN) @require_repo_read @anon_protect def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref): - manifest = v2.get_manifest_by_tag(namespace_name, repo_name, manifest_ref) + manifest = model.get_manifest_by_tag(namespace_name, repo_name, manifest_ref) if manifest is None: - has_tag = v2.has_active_tag(namespace_name, repo_name, manifest_ref) + has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref) if not has_tag: raise ManifestUnknown() @@ -47,7 +47,7 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref): if manifest is None: raise ManifestUnknown() - repo = v2.get_repository(namespace_name, repo_name) + repo = model.get_repository(namespace_name, repo_name) if repo is not None: track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2']) @@ -65,12 +65,12 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref): @require_repo_read @anon_protect def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref): - manifest = v2.get_manifest_by_digest(namespace_name, repo_name, manifest_ref) + manifest = model.get_manifest_by_digest(namespace_name, repo_name, manifest_ref) if manifest is None: # Without a tag name to reference, we can't make an attempt to generate the manifest raise ManifestUnknown() - repo = v2.get_repository(namespace_name, repo_name) + repo = model.get_repository(namespace_name, repo_name) if repo is not None: track_and_log('pull_repo', repo) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2']) @@ -137,7 +137,7 @@ def _write_manifest(namespace_name, repo_name, manifest): raise NameInvalid() # Ensure that the repository exists. - repo = v2.get_repository(namespace_name, repo_name) + repo = model.get_repository(namespace_name, repo_name) if repo is None: raise NameInvalid() @@ -145,7 +145,7 @@ def _write_manifest(namespace_name, repo_name, manifest): raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'}) # Ensure all the blobs in the manifest exist. - storage_map = v2.lookup_blobs_by_digest(namespace_name, repo_name, manifest.checksums) + storage_map = model.lookup_blobs_by_digest(namespace_name, repo_name, manifest.checksums) for layer in manifest.layers: digest_str = str(layer.digest) if digest_str not in storage_map: @@ -154,13 +154,13 @@ def _write_manifest(namespace_name, repo_name, manifest): # Lookup all the images and their parent images (if any) inside the manifest. # This will let us know which v1 images we need to synthesize and which ones are invalid. all_image_ids = list(manifest.parent_image_ids | manifest.image_ids) - images_map = v2.get_docker_v1_metadata_by_image_id(namespace_name, repo_name, all_image_ids) + images_map = model.get_docker_v1_metadata_by_image_id(namespace_name, repo_name, all_image_ids) # Rewrite any v1 image IDs that do not match the checksum in the database. try: rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map)) for rewritten_image in rewritten_images: - v1_metadata = v2.synthesize_v1_image( + model.synthesize_v1_image( repo, storage_map[rewritten_image.content_checksum], rewritten_image.image_id, @@ -175,8 +175,8 @@ def _write_manifest(namespace_name, repo_name, manifest): # Store the manifest pointing to the tag. leaf_layer_id = rewritten_images[-1].image_id - v2.save_manifest(namespace_name, repo_name, manifest.tag, leaf_layer_id, manifest.digest, - manifest.bytes) + model.save_manifest(namespace_name, repo_name, manifest.tag, leaf_layer_id, manifest.digest, + manifest.bytes) # Queue all blob manifests for replication. # TODO(jschorr): Find a way to optimize this insertion. @@ -213,7 +213,7 @@ def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref): Note: there is no equivalent method for deleting by tag name because it is forbidden by the spec. """ - tags = v2.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref) + tags = model.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref) if not tags: raise ManifestUnknown() @@ -225,9 +225,9 @@ def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref): def _generate_and_store_manifest(namespace_name, repo_name, tag_name): # Find the v1 metadata for this image and its parents. - v1_metadata = v2.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name) - parents_v1_metadata = v2.get_parents_docker_v1_metadata(namespace_name, repo_name, - v1_metadata.image_id) + v1_metadata = model.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name) + parents_v1_metadata = model.get_parents_docker_v1_metadata(namespace_name, repo_name, + v1_metadata.image_id) # If the manifest is being generated under the library namespace, then we make its namespace # empty. @@ -248,6 +248,6 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name): manifest = builder.build(docker_v2_signing_key) # Write the manifest to the DB. - v2.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, - manifest.bytes) + model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, + manifest.bytes) return manifest diff --git a/endpoints/v2/tag.py b/endpoints/v2/tag.py index 8fd3d32ab..6f2180b7c 100644 --- a/endpoints/v2/tag.py +++ b/endpoints/v2/tag.py @@ -5,7 +5,7 @@ from endpoints.common import parse_repository_name from endpoints.v2 import v2_bp, require_repo_read, paginate from endpoints.v2.errors import NameUnknown from endpoints.decorators import anon_protect -from data.interfaces import v2 +from data.interfaces.v2 import PreOCIModel as model @v2_bp.route('//tags/list', methods=['GET']) @parse_repository_name() @@ -14,11 +14,11 @@ from data.interfaces import v2 @anon_protect @paginate() def list_all_tags(namespace_name, repo_name, limit, offset, pagination_callback): - repo = v2.get_repository(namespace_name, repo_name) + repo = model.get_repository(namespace_name, repo_name) if repo is None: raise NameUnknown() - tags = v2.repository_tags(namespace_name, repo_name, limit, offset) + tags = model.repository_tags(namespace_name, repo_name, limit, offset) response = jsonify({ 'name': '{0}/{1}'.format(namespace_name, repo_name), 'tags': [tag.name for tag in tags], diff --git a/endpoints/v2/v2auth.py b/endpoints/v2/v2auth.py index 2398bde99..14e0b93fa 100644 --- a/endpoints/v2/v2auth.py +++ b/endpoints/v2/v2auth.py @@ -11,7 +11,7 @@ from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermissi CreateRepositoryPermission) from endpoints.v2 import v2_bp from endpoints.decorators import anon_protect -from data.interfaces import v2 +from data.interfaces.v2 import PreOCIModel as model from util.cache import no_cache from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX from util.security.registry_jwt import generate_bearer_token, build_context_and_subject @@ -96,7 +96,7 @@ def generate_registry_jwt(): if user is not None or token is not None: # Lookup the repository. If it exists, make sure the entity has modify # permission. Otherwise, make sure the entity has create permission. - repo = v2.get_repository(namespace, reponame) + repo = model.get_repository(namespace, reponame) if repo: if ModifyRepositoryPermission(namespace, reponame).can(): final_actions.append('push') @@ -105,7 +105,7 @@ def generate_registry_jwt(): else: if CreateRepositoryPermission(namespace).can() and user is not None: logger.debug('Creating repository: %s/%s', namespace, reponame) - v2.create_repository(namespace, reponame, user) + model.create_repository(namespace, reponame, user) final_actions.append('push') else: logger.debug('No permission to create repository %s/%s', namespace, reponame) @@ -113,7 +113,7 @@ def generate_registry_jwt(): if 'pull' in actions: # Grant pull if the user can read the repo or it is public. if (ReadRepositoryPermission(namespace, reponame).can() or - v2.repository_is_public(namespace, reponame)): + model.repository_is_public(namespace, reponame)): final_actions.append('pull') else: logger.debug('No permission to pull repository %s/%s', namespace, reponame) diff --git a/image/__init__.py b/image/__init__.py index 1ea1f42bb..e69de29bb 100644 --- a/image/__init__.py +++ b/image/__init__.py @@ -1,111 +0,0 @@ -import tarfile - -from collections import namedtuple -from namedlist import namedlist - -from util.registry.gzipwrap import GzipWrap - - -class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])): - """ - ManifestJSON represents a Manifest of any format. - """ - -class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])): - """ - RepositoryReference represents a reference to a Repository, without its full metadata. - """ - - -class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', - 'is_public'])): - """ - Repository represents a collection of tags. - """ - - -class Tag(namedtuple('Tag', ['name', 'repository'])): - """ - Tag represents a user-facing alias for referencing a set of Manifests. - """ - - -class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count', - 'chunk_count', 'sha_state', 'location_name', - 'storage_metadata', 'piece_sha_state', 'piece_hashes', - 'repo_namespace_name', 'repo_name'])): - """ - BlobUpload represents the current state of an Blob being uploaded. - """ - - -class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])): - """ - Blob represents an opaque binary blob saved to the storage system. - """ - - -class TarImageFormatter(object): - """ - Base class for classes which produce a tar containing image and layer data. - """ - - def build_stream(self, namespace, repository, tag, synthetic_image_id, layer_json, - get_image_iterator, get_layer_iterator, get_image_json): - """ - Builds and streams a synthetic .tar.gz that represents the formatted tar created by this class's - implementation. - """ - return GzipWrap(self.stream_generator(namespace, repository, tag, - synthetic_image_id, layer_json, - get_image_iterator, get_layer_iterator, - get_image_json)) - - def stream_generator(self, namespace, repository, tag, synthetic_image_id, - layer_json, get_image_iterator, get_layer_iterator, get_image_json): - raise NotImplementedError - - def tar_file(self, name, contents, mtime=None): - """ - Returns the tar binary representation for a file with the given name and file contents. - """ - length = len(contents) - tar_data = self.tar_file_header(name, length, mtime=mtime) - tar_data += contents - tar_data += self.tar_file_padding(length) - return tar_data - - def tar_file_padding(self, length): - """ - Returns tar file padding for file data of the given length. - """ - if length % 512 != 0: - return '\0' * (512 - (length % 512)) - - return '' - - def tar_file_header(self, name, file_size, mtime=None): - """ - Returns tar file header data for a file with the given name and size. - """ - info = tarfile.TarInfo(name=name) - info.type = tarfile.REGTYPE - info.size = file_size - - if mtime is not None: - info.mtime = mtime - return info.tobuf() - - def tar_folder(self, name, mtime=None): - """ - Returns tar file header data for a folder with the given name. - """ - info = tarfile.TarInfo(name=name) - info.type = tarfile.DIRTYPE - - if mtime is not None: - info.mtime = mtime - - # allow the directory to be readable by non-root users - info.mode = 0755 - return info.tobuf() diff --git a/image/appc/__init__.py b/image/appc/__init__.py index 592825e43..e26f0d3e6 100644 --- a/image/appc/__init__.py +++ b/image/appc/__init__.py @@ -6,7 +6,7 @@ from uuid import uuid4 from app import app from util.registry.streamlayerformat import StreamLayerMerger -from image import TarImageFormatter +from image.common import TarImageFormatter ACNAME_REGEX = re.compile(r'[^a-z-]+') diff --git a/image/common.py b/image/common.py new file mode 100644 index 000000000..28b628abf --- /dev/null +++ b/image/common.py @@ -0,0 +1,68 @@ +import tarfile +from util.registry.gzipwrap import GzipWrap + + +class TarImageFormatter(object): + """ + Base class for classes which produce a tar containing image and layer data. + """ + + def build_stream(self, namespace, repository, tag, synthetic_image_id, layer_json, + get_image_iterator, get_layer_iterator, get_image_json): + """ + Builds and streams a synthetic .tar.gz that represents the formatted tar created by this class's + implementation. + """ + return GzipWrap(self.stream_generator(namespace, repository, tag, + synthetic_image_id, layer_json, + get_image_iterator, get_layer_iterator, + get_image_json)) + + def stream_generator(self, namespace, repository, tag, synthetic_image_id, + layer_json, get_image_iterator, get_layer_iterator, get_image_json): + raise NotImplementedError + + def tar_file(self, name, contents, mtime=None): + """ + Returns the tar binary representation for a file with the given name and file contents. + """ + length = len(contents) + tar_data = self.tar_file_header(name, length, mtime=mtime) + tar_data += contents + tar_data += self.tar_file_padding(length) + return tar_data + + def tar_file_padding(self, length): + """ + Returns tar file padding for file data of the given length. + """ + if length % 512 != 0: + return '\0' * (512 - (length % 512)) + + return '' + + def tar_file_header(self, name, file_size, mtime=None): + """ + Returns tar file header data for a file with the given name and size. + """ + info = tarfile.TarInfo(name=name) + info.type = tarfile.REGTYPE + info.size = file_size + + if mtime is not None: + info.mtime = mtime + return info.tobuf() + + def tar_folder(self, name, mtime=None): + """ + Returns tar file header data for a folder with the given name. + """ + info = tarfile.TarInfo(name=name) + info.type = tarfile.DIRTYPE + + if mtime is not None: + info.mtime = mtime + + # allow the directory to be readable by non-root users + info.mode = 0755 + return info.tobuf() diff --git a/image/docker/squashed.py b/image/docker/squashed.py index d3c886185..bf209eb1e 100644 --- a/image/docker/squashed.py +++ b/image/docker/squashed.py @@ -4,7 +4,7 @@ import math import calendar from app import app -from image import TarImageFormatter +from image.common import TarImageFormatter from util.registry.gzipwrap import GZIP_BUFFER_SIZE from util.registry.streamlayerformat import StreamLayerMerger