diff --git a/endpoints/v2/__init__.py b/endpoints/v2/__init__.py index 1b63c6f52..a2b95e972 100644 --- a/endpoints/v2/__init__.py +++ b/endpoints/v2/__init__.py @@ -15,9 +15,9 @@ from auth.auth_context import get_grant_context from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, AdministerRepositoryPermission) from auth.registry_jwt_auth import process_registry_jwt_auth, get_auth_headers -from data.interfaces.v2 import pre_oci_model as model from endpoints.decorators import anon_protect, anon_allowed from endpoints.v2.errors import V2RegistryException, Unauthorized, Unsupported, NameUnknown +from endpoints.v2.models_pre_oci import data_model as model from util.http import abort from util.metrics.metricqueue import time_blueprint from util.registry.dockerver import docker_version diff --git a/endpoints/v2/blob.py b/endpoints/v2/blob.py index cd77ee2ee..d8b361318 100644 --- a/endpoints/v2/blob.py +++ b/endpoints/v2/blob.py @@ -10,13 +10,13 @@ import resumablehashlib from app import storage, app, get_app_url, metric_queue from auth.registry_jwt_auth import process_registry_jwt_auth from data import database -from data.interfaces.v2 import pre_oci_model as model from digest import digest_tools from endpoints.common import parse_repository_name +from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream from endpoints.v2.errors import (BlobUnknown, BlobUploadInvalid, BlobUploadUnknown, Unsupported, NameUnknown, LayerTooLarge) -from endpoints.decorators import anon_protect +from endpoints.v2.models_pre_oci import data_model as model from util.cache import cache_control from util.registry.filelike import wrap_with_handler, StreamSlice from util.registry.gzipstream import calculate_size_handler diff --git a/endpoints/v2/catalog.py b/endpoints/v2/catalog.py index 18c27db82..01720bce4 100644 --- a/endpoints/v2/catalog.py +++ b/endpoints/v2/catalog.py @@ -5,7 +5,7 @@ from flask import jsonify from auth.registry_jwt_auth import process_registry_jwt_auth, get_granted_entity from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp, paginate -from data.interfaces.v2 import pre_oci_model as model +from endpoints.v2.models_pre_oci import data_model as model @v2_bp.route('/_catalog', methods=['GET']) @process_registry_jwt_auth() diff --git a/endpoints/v2/manifest.py b/endpoints/v2/manifest.py index 732403598..09c70e169 100644 --- a/endpoints/v2/manifest.py +++ b/endpoints/v2/manifest.py @@ -8,14 +8,15 @@ import features from app import docker_v2_signing_key, app, metric_queue from auth.registry_jwt_auth import process_registry_jwt_auth -from data.interfaces.v2 import pre_oci_model as model, Label from digest import digest_tools from endpoints.common import parse_repository_name from endpoints.decorators import anon_protect -from endpoints.v2 import v2_bp, require_repo_read, require_repo_write -from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, - NameInvalid) from endpoints.notificationhelper import spawn_notification +from endpoints.v2 import v2_bp, require_repo_read, require_repo_write +from endpoints.v2.errors import ( + BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid) +from endpoints.v2.models_interface import Label +from endpoints.v2.models_pre_oci import data_model as model from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES diff --git a/endpoints/v2/models_interface.py b/endpoints/v2/models_interface.py new file mode 100644 index 000000000..1904871f3 --- /dev/null +++ b/endpoints/v2/models_interface.py @@ -0,0 +1,255 @@ +from abc import ABCMeta, abstractmethod +from collections import namedtuple + +from namedlist import namedlist +from six import add_metaclass + + +class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', + 'is_public', 'kind', 'trust_enabled'])): + """ + Repository represents a namespaced collection of tags. + :type id: int + :type name: string + :type namespace_name: string + :type description: string + :type is_public: bool + :type kind: string + :type trust_enabled: bool + """ + +class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])): + """ + ManifestJSON represents a Manifest of any format. + """ + + +class Tag(namedtuple('Tag', ['name', 'repository'])): + """ + Tag represents a user-facing alias for referencing a set of Manifests. + """ + + +class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count', + 'chunk_count', 'sha_state', 'location_name', + 'storage_metadata', 'piece_sha_state', 'piece_hashes', + 'repo_namespace_name', 'repo_name'])): + """ + BlobUpload represents the current state of an Blob being uploaded. + """ + + +class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])): + """ + Blob represents an opaque binary blob saved to the storage system. + """ + + +class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])): + """ + RepositoryReference represents a reference to a Repository, without its full metadata. + """ + +class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])): + """ + Label represents a key-value pair that describes a particular Manifest. + """ + + +@add_metaclass(ABCMeta) +class DockerRegistryV2DataInterface(object): + """ + Interface that represents all data store interactions required by a Docker Registry v1. + """ + + @abstractmethod + def create_repository(self, namespace_name, repo_name, creating_user=None): + """ + Creates a new repository under the specified namespace with the given name. The user supplied is + the user creating the repository, if any. + """ + pass + + @abstractmethod + def get_repository(self, namespace_name, repo_name): + """ + Returns a repository tuple for the repository with the given name under the given namespace. + Returns None if no such repository was found. + """ + pass + + @abstractmethod + def has_active_tag(self, namespace_name, repo_name, tag_name): + """ + Returns whether there is an active tag for the tag with the given name under the matching + repository, if any, or none if none. + """ + pass + + @abstractmethod + def get_manifest_by_tag(self, namespace_name, repo_name, tag_name): + """ + Returns the current manifest for the tag with the given name under the matching repository, if + any, or None if none. + """ + pass + + @abstractmethod + def get_manifest_by_digest(self, namespace_name, repo_name, digest): + """ + Returns the manifest matching the given digest under the matching repository, if any, or None if + none. + """ + pass + + @abstractmethod + def delete_manifest_by_digest(self, namespace_name, repo_name, digest): + """ + Deletes the manifest with the associated digest (if any) and returns all removed tags that + pointed to that manifest. If the manifest was not found, returns an empty list. + """ + pass + + @abstractmethod + def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name): + """ + Returns the Docker V1 metadata associated with the tag with the given name under the matching + repository, if any. If none, returns None. + """ + pass + + @abstractmethod + def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids): + """ + Returns a map of Docker V1 metadata for each given image ID, matched under the repository with + the given namespace and name. Returns an empty map if the matching repository was not found. + """ + pass + + @abstractmethod + def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id): + """ + Returns an ordered list containing the Docker V1 metadata for each parent of the image with the + given docker ID under the matching repository. Returns an empty list if the image was not found. + """ + pass + + @abstractmethod + def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest, + manifest_bytes): + """ + Creates a new manifest with the given digest and byte data, and assigns the tag with the given + name under the matching repository to it. + """ + pass + + @abstractmethod + def synthesize_v1_image(self, repository, storage, image_id, created, comment, command, + compat_json, parent_image_id): + """ + Synthesizes a V1 image under the specified repository, pointing to the given storage and returns + the V1 metadata for the synthesized image. + """ + pass + + @abstractmethod + def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id, + manifest_digest, manifest_bytes): + """ + Saves a manifest pointing to the given leaf image, with the given manifest, under the matching + repository as a tag with the given name. + + Returns a boolean whether or not the tag was newly created or not. + """ + pass + + @abstractmethod + def repository_tags(self, namespace_name, repo_name, limit, offset): + """ + Returns the active tags under the repository with the given name and namespace. + """ + pass + + @abstractmethod + def get_visible_repositories(self, username, limit, offset): + """ + Returns the repositories visible to the user with the given username, if any. + """ + pass + + @abstractmethod + def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata): + """ + Creates a blob upload under the matching repository with the given UUID and metadata. + Returns whether the matching repository exists. + """ + pass + + @abstractmethod + def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid): + """ + Searches for a blob upload with the given UUID under the given repository and returns it or None + if none. + """ + pass + + @abstractmethod + def update_blob_upload(self, blob_upload): + """ + Saves any changes to the blob upload object given to the backing data store. + Fields that can change: + - uncompressed_byte_count + - piece_hashes + - piece_sha_state + - storage_metadata + - byte_count + - chunk_count + - sha_state + """ + pass + + @abstractmethod + def delete_blob_upload(self, namespace_name, repo_name, uuid): + """ + Deletes the blob upload with the given uuid under the matching repository. If none, does + nothing. + """ + pass + + @abstractmethod + def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload, + expiration_sec): + """ + Creates a blob and links a temporary tag with the specified expiration to it under the matching + repository. + """ + pass + + @abstractmethod + def get_blob_by_digest(self, namespace_name, repo_name, digest): + """ + Returns the blob with the given digest under the matching repository or None if none. + """ + pass + + @abstractmethod + def save_bittorrent_pieces(self, blob, piece_size, piece_bytes): + """ + Saves the BitTorrent piece hashes for the given blob. + """ + pass + + @abstractmethod + def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels): + """ + Creates a new labels for the provided manifest. + """ + pass + + + @abstractmethod + def get_blob_path(self, blob): + """ + Once everything is moved over, this could be in util.registry and not even touch the database. + """ + pass diff --git a/data/interfaces/v2.py b/endpoints/v2/models_pre_oci.py similarity index 59% rename from data/interfaces/v2.py rename to endpoints/v2/models_pre_oci.py index 56f778bb8..3a963b3fb 100644 --- a/data/interfaces/v2.py +++ b/endpoints/v2/models_pre_oci.py @@ -1,267 +1,22 @@ -from abc import ABCMeta, abstractmethod -from collections import namedtuple - -from namedlist import namedlist from peewee import IntegrityError -from six import add_metaclass from data import model, database from data.model import DataModelException +from endpoints.v2.models_interface import ( + Blob, + BlobUpload, + DockerRegistryV2DataInterface, + ManifestJSON, + Repository, + RepositoryReference, + Tag, +) from image.docker.v1 import DockerV1Metadata + _MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws" -class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', - 'is_public', 'kind', 'trust_enabled'])): - """ - Repository represents a namespaced collection of tags. - :type id: int - :type name: string - :type namespace_name: string - :type description: string - :type is_public: bool - :type kind: string - :type trust_enabled: bool - """ - -class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])): - """ - ManifestJSON represents a Manifest of any format. - """ - - -class Tag(namedtuple('Tag', ['name', 'repository'])): - """ - Tag represents a user-facing alias for referencing a set of Manifests. - """ - - -class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count', - 'chunk_count', 'sha_state', 'location_name', - 'storage_metadata', 'piece_sha_state', 'piece_hashes', - 'repo_namespace_name', 'repo_name'])): - """ - BlobUpload represents the current state of an Blob being uploaded. - """ - - -class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])): - """ - Blob represents an opaque binary blob saved to the storage system. - """ - - -class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])): - """ - RepositoryReference represents a reference to a Repository, without its full metadata. - """ - -class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])): - """ - Label represents a key-value pair that describes a particular Manifest. - """ - - -@add_metaclass(ABCMeta) -class DockerRegistryV2DataInterface(object): - """ - Interface that represents all data store interactions required by a Docker Registry v1. - """ - - @abstractmethod - def create_repository(self, namespace_name, repo_name, creating_user=None): - """ - Creates a new repository under the specified namespace with the given name. The user supplied is - the user creating the repository, if any. - """ - pass - - @abstractmethod - def get_repository(self, namespace_name, repo_name): - """ - Returns a repository tuple for the repository with the given name under the given namespace. - Returns None if no such repository was found. - """ - pass - - @abstractmethod - def has_active_tag(self, namespace_name, repo_name, tag_name): - """ - Returns whether there is an active tag for the tag with the given name under the matching - repository, if any, or none if none. - """ - pass - - @abstractmethod - def get_manifest_by_tag(self, namespace_name, repo_name, tag_name): - """ - Returns the current manifest for the tag with the given name under the matching repository, if - any, or None if none. - """ - pass - - @abstractmethod - def get_manifest_by_digest(self, namespace_name, repo_name, digest): - """ - Returns the manifest matching the given digest under the matching repository, if any, or None if - none. - """ - pass - - @abstractmethod - def delete_manifest_by_digest(self, namespace_name, repo_name, digest): - """ - Deletes the manifest with the associated digest (if any) and returns all removed tags that - pointed to that manifest. If the manifest was not found, returns an empty list. - """ - pass - - @abstractmethod - def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name): - """ - Returns the Docker V1 metadata associated with the tag with the given name under the matching - repository, if any. If none, returns None. - """ - pass - - @abstractmethod - def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids): - """ - Returns a map of Docker V1 metadata for each given image ID, matched under the repository with - the given namespace and name. Returns an empty map if the matching repository was not found. - """ - pass - - @abstractmethod - def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id): - """ - Returns an ordered list containing the Docker V1 metadata for each parent of the image with the - given docker ID under the matching repository. Returns an empty list if the image was not found. - """ - pass - - @abstractmethod - def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest, - manifest_bytes): - """ - Creates a new manifest with the given digest and byte data, and assigns the tag with the given - name under the matching repository to it. - """ - pass - - @abstractmethod - def synthesize_v1_image(self, repository, storage, image_id, created, comment, command, - compat_json, parent_image_id): - """ - Synthesizes a V1 image under the specified repository, pointing to the given storage and returns - the V1 metadata for the synthesized image. - """ - pass - - @abstractmethod - def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id, - manifest_digest, manifest_bytes): - """ - Saves a manifest pointing to the given leaf image, with the given manifest, under the matching - repository as a tag with the given name. - - Returns a boolean whether or not the tag was newly created or not. - """ - pass - - @abstractmethod - def repository_tags(self, namespace_name, repo_name, limit, offset): - """ - Returns the active tags under the repository with the given name and namespace. - """ - pass - - @abstractmethod - def get_visible_repositories(self, username, limit, offset): - """ - Returns the repositories visible to the user with the given username, if any. - """ - pass - - @abstractmethod - def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata): - """ - Creates a blob upload under the matching repository with the given UUID and metadata. - Returns whether the matching repository exists. - """ - pass - - @abstractmethod - def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid): - """ - Searches for a blob upload with the given UUID under the given repository and returns it or None - if none. - """ - pass - - @abstractmethod - def update_blob_upload(self, blob_upload): - """ - Saves any changes to the blob upload object given to the backing data store. - Fields that can change: - - uncompressed_byte_count - - piece_hashes - - piece_sha_state - - storage_metadata - - byte_count - - chunk_count - - sha_state - """ - pass - - @abstractmethod - def delete_blob_upload(self, namespace_name, repo_name, uuid): - """ - Deletes the blob upload with the given uuid under the matching repository. If none, does - nothing. - """ - pass - - @abstractmethod - def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload, - expiration_sec): - """ - Creates a blob and links a temporary tag with the specified expiration to it under the matching - repository. - """ - pass - - @abstractmethod - def get_blob_by_digest(self, namespace_name, repo_name, digest): - """ - Returns the blob with the given digest under the matching repository or None if none. - """ - pass - - @abstractmethod - def save_bittorrent_pieces(self, blob, piece_size, piece_bytes): - """ - Saves the BitTorrent piece hashes for the given blob. - """ - pass - - @abstractmethod - def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels): - """ - Creates a new labels for the provided manifest. - """ - pass - - - @abstractmethod - def get_blob_path(self, blob): - """ - Once everything is moved over, this could be in util.registry and not even touch the database. - """ - pass - - class PreOCIModel(DockerRegistryV2DataInterface): """ PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema @@ -544,4 +299,4 @@ def _repository_for_repo(repo): ) -pre_oci_model = PreOCIModel() +data_model = PreOCIModel() diff --git a/endpoints/v2/tag.py b/endpoints/v2/tag.py index 683480ac2..9c0e81b02 100644 --- a/endpoints/v2/tag.py +++ b/endpoints/v2/tag.py @@ -2,9 +2,9 @@ from flask import jsonify from auth.registry_jwt_auth import process_registry_jwt_auth from endpoints.common import parse_repository_name -from endpoints.v2 import v2_bp, require_repo_read, paginate from endpoints.decorators import anon_protect -from data.interfaces.v2 import pre_oci_model as model +from endpoints.v2 import v2_bp, require_repo_read, paginate +from endpoints.v2.models_pre_oci import data_model as model @v2_bp.route('//tags/list', methods=['GET']) @parse_repository_name() diff --git a/endpoints/v2/v2auth.py b/endpoints/v2/v2auth.py index 0d9e8ffb0..747404b88 100644 --- a/endpoints/v2/v2auth.py +++ b/endpoints/v2/v2auth.py @@ -13,10 +13,10 @@ from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermissi from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp from endpoints.v2.errors import InvalidLogin, NameInvalid, InvalidRequest, Unsupported, Unauthorized -from data.interfaces.v2 import pre_oci_model as model +from endpoints.v2.models_pre_oci import data_model as model from util.cache import no_cache from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX -from util.security.registry_jwt import (generate_bearer_token, build_context_and_subject, QUAY_TUF_ROOT, +from util.security.registry_jwt import (generate_bearer_token, build_context_and_subject, QUAY_TUF_ROOT, SIGNER_TUF_ROOT, DISABLED_TUF_ROOT) logger = logging.getLogger(__name__) @@ -188,7 +188,7 @@ def generate_registry_jwt(auth_result): def get_tuf_root(repo, namespace, reponame): if not features.SIGNING or repo is None or not repo.trust_enabled: return DISABLED_TUF_ROOT - + # Users with write access to a repo will see signer-rooted TUF metadata if ModifyRepositoryPermission(namespace, reponame).can(): return SIGNER_TUF_ROOT