endpoints.v2: new fs layout for data interface

Fixes QUAY-658
This commit is contained in:
Jimmy Zelinskie 2017-06-26 18:10:39 -04:00
parent 7efd222a19
commit 0e26a03f7e
8 changed files with 280 additions and 269 deletions

View file

@ -15,9 +15,9 @@ from auth.auth_context import get_grant_context
from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
AdministerRepositoryPermission)
from auth.registry_jwt_auth import process_registry_jwt_auth, get_auth_headers
from data.interfaces.v2 import pre_oci_model as model
from endpoints.decorators import anon_protect, anon_allowed
from endpoints.v2.errors import V2RegistryException, Unauthorized, Unsupported, NameUnknown
from endpoints.v2.models_pre_oci import data_model as model
from util.http import abort
from util.metrics.metricqueue import time_blueprint
from util.registry.dockerver import docker_version

View file

@ -10,13 +10,13 @@ import resumablehashlib
from app import storage, app, get_app_url, metric_queue
from auth.registry_jwt_auth import process_registry_jwt_auth
from data import database
from data.interfaces.v2 import pre_oci_model as model
from digest import digest_tools
from endpoints.common import parse_repository_name
from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream
from endpoints.v2.errors import (BlobUnknown, BlobUploadInvalid, BlobUploadUnknown, Unsupported,
NameUnknown, LayerTooLarge)
from endpoints.decorators import anon_protect
from endpoints.v2.models_pre_oci import data_model as model
from util.cache import cache_control
from util.registry.filelike import wrap_with_handler, StreamSlice
from util.registry.gzipstream import calculate_size_handler

View file

@ -5,7 +5,7 @@ from flask import jsonify
from auth.registry_jwt_auth import process_registry_jwt_auth, get_granted_entity
from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, paginate
from data.interfaces.v2 import pre_oci_model as model
from endpoints.v2.models_pre_oci import data_model as model
@v2_bp.route('/_catalog', methods=['GET'])
@process_registry_jwt_auth()

View file

@ -8,14 +8,15 @@ import features
from app import docker_v2_signing_key, app, metric_queue
from auth.registry_jwt_auth import process_registry_jwt_auth
from data.interfaces.v2 import pre_oci_model as model, Label
from digest import digest_tools
from endpoints.common import parse_repository_name
from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
NameInvalid)
from endpoints.notificationhelper import spawn_notification
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
from endpoints.v2.errors import (
BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid)
from endpoints.v2.models_interface import Label
from endpoints.v2.models_pre_oci import data_model as model
from image.docker import ManifestException
from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES

View file

@ -0,0 +1,255 @@
from abc import ABCMeta, abstractmethod
from collections import namedtuple
from namedlist import namedlist
from six import add_metaclass
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public', 'kind', 'trust_enabled'])):
"""
Repository represents a namespaced collection of tags.
:type id: int
:type name: string
:type namespace_name: string
:type description: string
:type is_public: bool
:type kind: string
:type trust_enabled: bool
"""
class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
"""
ManifestJSON represents a Manifest of any format.
"""
class Tag(namedtuple('Tag', ['name', 'repository'])):
"""
Tag represents a user-facing alias for referencing a set of Manifests.
"""
class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count',
'chunk_count', 'sha_state', 'location_name',
'storage_metadata', 'piece_sha_state', 'piece_hashes',
'repo_namespace_name', 'repo_name'])):
"""
BlobUpload represents the current state of an Blob being uploaded.
"""
class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])):
"""
Blob represents an opaque binary blob saved to the storage system.
"""
class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
"""
RepositoryReference represents a reference to a Repository, without its full metadata.
"""
class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])):
"""
Label represents a key-value pair that describes a particular Manifest.
"""
@add_metaclass(ABCMeta)
class DockerRegistryV2DataInterface(object):
"""
Interface that represents all data store interactions required by a Docker Registry v1.
"""
@abstractmethod
def create_repository(self, namespace_name, repo_name, creating_user=None):
"""
Creates a new repository under the specified namespace with the given name. The user supplied is
the user creating the repository, if any.
"""
pass
@abstractmethod
def get_repository(self, namespace_name, repo_name):
"""
Returns a repository tuple for the repository with the given name under the given namespace.
Returns None if no such repository was found.
"""
pass
@abstractmethod
def has_active_tag(self, namespace_name, repo_name, tag_name):
"""
Returns whether there is an active tag for the tag with the given name under the matching
repository, if any, or none if none.
"""
pass
@abstractmethod
def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
"""
Returns the current manifest for the tag with the given name under the matching repository, if
any, or None if none.
"""
pass
@abstractmethod
def get_manifest_by_digest(self, namespace_name, repo_name, digest):
"""
Returns the manifest matching the given digest under the matching repository, if any, or None if
none.
"""
pass
@abstractmethod
def delete_manifest_by_digest(self, namespace_name, repo_name, digest):
"""
Deletes the manifest with the associated digest (if any) and returns all removed tags that
pointed to that manifest. If the manifest was not found, returns an empty list.
"""
pass
@abstractmethod
def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name):
"""
Returns the Docker V1 metadata associated with the tag with the given name under the matching
repository, if any. If none, returns None.
"""
pass
@abstractmethod
def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids):
"""
Returns a map of Docker V1 metadata for each given image ID, matched under the repository with
the given namespace and name. Returns an empty map if the matching repository was not found.
"""
pass
@abstractmethod
def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id):
"""
Returns an ordered list containing the Docker V1 metadata for each parent of the image with the
given docker ID under the matching repository. Returns an empty list if the image was not found.
"""
pass
@abstractmethod
def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest,
manifest_bytes):
"""
Creates a new manifest with the given digest and byte data, and assigns the tag with the given
name under the matching repository to it.
"""
pass
@abstractmethod
def synthesize_v1_image(self, repository, storage, image_id, created, comment, command,
compat_json, parent_image_id):
"""
Synthesizes a V1 image under the specified repository, pointing to the given storage and returns
the V1 metadata for the synthesized image.
"""
pass
@abstractmethod
def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id,
manifest_digest, manifest_bytes):
"""
Saves a manifest pointing to the given leaf image, with the given manifest, under the matching
repository as a tag with the given name.
Returns a boolean whether or not the tag was newly created or not.
"""
pass
@abstractmethod
def repository_tags(self, namespace_name, repo_name, limit, offset):
"""
Returns the active tags under the repository with the given name and namespace.
"""
pass
@abstractmethod
def get_visible_repositories(self, username, limit, offset):
"""
Returns the repositories visible to the user with the given username, if any.
"""
pass
@abstractmethod
def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata):
"""
Creates a blob upload under the matching repository with the given UUID and metadata.
Returns whether the matching repository exists.
"""
pass
@abstractmethod
def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid):
"""
Searches for a blob upload with the given UUID under the given repository and returns it or None
if none.
"""
pass
@abstractmethod
def update_blob_upload(self, blob_upload):
"""
Saves any changes to the blob upload object given to the backing data store.
Fields that can change:
- uncompressed_byte_count
- piece_hashes
- piece_sha_state
- storage_metadata
- byte_count
- chunk_count
- sha_state
"""
pass
@abstractmethod
def delete_blob_upload(self, namespace_name, repo_name, uuid):
"""
Deletes the blob upload with the given uuid under the matching repository. If none, does
nothing.
"""
pass
@abstractmethod
def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload,
expiration_sec):
"""
Creates a blob and links a temporary tag with the specified expiration to it under the matching
repository.
"""
pass
@abstractmethod
def get_blob_by_digest(self, namespace_name, repo_name, digest):
"""
Returns the blob with the given digest under the matching repository or None if none.
"""
pass
@abstractmethod
def save_bittorrent_pieces(self, blob, piece_size, piece_bytes):
"""
Saves the BitTorrent piece hashes for the given blob.
"""
pass
@abstractmethod
def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels):
"""
Creates a new labels for the provided manifest.
"""
pass
@abstractmethod
def get_blob_path(self, blob):
"""
Once everything is moved over, this could be in util.registry and not even touch the database.
"""
pass

View file

@ -1,267 +1,22 @@
from abc import ABCMeta, abstractmethod
from collections import namedtuple
from namedlist import namedlist
from peewee import IntegrityError
from six import add_metaclass
from data import model, database
from data.model import DataModelException
from endpoints.v2.models_interface import (
Blob,
BlobUpload,
DockerRegistryV2DataInterface,
ManifestJSON,
Repository,
RepositoryReference,
Tag,
)
from image.docker.v1 import DockerV1Metadata
_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public', 'kind', 'trust_enabled'])):
"""
Repository represents a namespaced collection of tags.
:type id: int
:type name: string
:type namespace_name: string
:type description: string
:type is_public: bool
:type kind: string
:type trust_enabled: bool
"""
class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
"""
ManifestJSON represents a Manifest of any format.
"""
class Tag(namedtuple('Tag', ['name', 'repository'])):
"""
Tag represents a user-facing alias for referencing a set of Manifests.
"""
class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count',
'chunk_count', 'sha_state', 'location_name',
'storage_metadata', 'piece_sha_state', 'piece_hashes',
'repo_namespace_name', 'repo_name'])):
"""
BlobUpload represents the current state of an Blob being uploaded.
"""
class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])):
"""
Blob represents an opaque binary blob saved to the storage system.
"""
class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
"""
RepositoryReference represents a reference to a Repository, without its full metadata.
"""
class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])):
"""
Label represents a key-value pair that describes a particular Manifest.
"""
@add_metaclass(ABCMeta)
class DockerRegistryV2DataInterface(object):
"""
Interface that represents all data store interactions required by a Docker Registry v1.
"""
@abstractmethod
def create_repository(self, namespace_name, repo_name, creating_user=None):
"""
Creates a new repository under the specified namespace with the given name. The user supplied is
the user creating the repository, if any.
"""
pass
@abstractmethod
def get_repository(self, namespace_name, repo_name):
"""
Returns a repository tuple for the repository with the given name under the given namespace.
Returns None if no such repository was found.
"""
pass
@abstractmethod
def has_active_tag(self, namespace_name, repo_name, tag_name):
"""
Returns whether there is an active tag for the tag with the given name under the matching
repository, if any, or none if none.
"""
pass
@abstractmethod
def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
"""
Returns the current manifest for the tag with the given name under the matching repository, if
any, or None if none.
"""
pass
@abstractmethod
def get_manifest_by_digest(self, namespace_name, repo_name, digest):
"""
Returns the manifest matching the given digest under the matching repository, if any, or None if
none.
"""
pass
@abstractmethod
def delete_manifest_by_digest(self, namespace_name, repo_name, digest):
"""
Deletes the manifest with the associated digest (if any) and returns all removed tags that
pointed to that manifest. If the manifest was not found, returns an empty list.
"""
pass
@abstractmethod
def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name):
"""
Returns the Docker V1 metadata associated with the tag with the given name under the matching
repository, if any. If none, returns None.
"""
pass
@abstractmethod
def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids):
"""
Returns a map of Docker V1 metadata for each given image ID, matched under the repository with
the given namespace and name. Returns an empty map if the matching repository was not found.
"""
pass
@abstractmethod
def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id):
"""
Returns an ordered list containing the Docker V1 metadata for each parent of the image with the
given docker ID under the matching repository. Returns an empty list if the image was not found.
"""
pass
@abstractmethod
def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest,
manifest_bytes):
"""
Creates a new manifest with the given digest and byte data, and assigns the tag with the given
name under the matching repository to it.
"""
pass
@abstractmethod
def synthesize_v1_image(self, repository, storage, image_id, created, comment, command,
compat_json, parent_image_id):
"""
Synthesizes a V1 image under the specified repository, pointing to the given storage and returns
the V1 metadata for the synthesized image.
"""
pass
@abstractmethod
def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id,
manifest_digest, manifest_bytes):
"""
Saves a manifest pointing to the given leaf image, with the given manifest, under the matching
repository as a tag with the given name.
Returns a boolean whether or not the tag was newly created or not.
"""
pass
@abstractmethod
def repository_tags(self, namespace_name, repo_name, limit, offset):
"""
Returns the active tags under the repository with the given name and namespace.
"""
pass
@abstractmethod
def get_visible_repositories(self, username, limit, offset):
"""
Returns the repositories visible to the user with the given username, if any.
"""
pass
@abstractmethod
def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata):
"""
Creates a blob upload under the matching repository with the given UUID and metadata.
Returns whether the matching repository exists.
"""
pass
@abstractmethod
def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid):
"""
Searches for a blob upload with the given UUID under the given repository and returns it or None
if none.
"""
pass
@abstractmethod
def update_blob_upload(self, blob_upload):
"""
Saves any changes to the blob upload object given to the backing data store.
Fields that can change:
- uncompressed_byte_count
- piece_hashes
- piece_sha_state
- storage_metadata
- byte_count
- chunk_count
- sha_state
"""
pass
@abstractmethod
def delete_blob_upload(self, namespace_name, repo_name, uuid):
"""
Deletes the blob upload with the given uuid under the matching repository. If none, does
nothing.
"""
pass
@abstractmethod
def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload,
expiration_sec):
"""
Creates a blob and links a temporary tag with the specified expiration to it under the matching
repository.
"""
pass
@abstractmethod
def get_blob_by_digest(self, namespace_name, repo_name, digest):
"""
Returns the blob with the given digest under the matching repository or None if none.
"""
pass
@abstractmethod
def save_bittorrent_pieces(self, blob, piece_size, piece_bytes):
"""
Saves the BitTorrent piece hashes for the given blob.
"""
pass
@abstractmethod
def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels):
"""
Creates a new labels for the provided manifest.
"""
pass
@abstractmethod
def get_blob_path(self, blob):
"""
Once everything is moved over, this could be in util.registry and not even touch the database.
"""
pass
class PreOCIModel(DockerRegistryV2DataInterface):
"""
PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema
@ -544,4 +299,4 @@ def _repository_for_repo(repo):
)
pre_oci_model = PreOCIModel()
data_model = PreOCIModel()

View file

@ -2,9 +2,9 @@ from flask import jsonify
from auth.registry_jwt_auth import process_registry_jwt_auth
from endpoints.common import parse_repository_name
from endpoints.v2 import v2_bp, require_repo_read, paginate
from endpoints.decorators import anon_protect
from data.interfaces.v2 import pre_oci_model as model
from endpoints.v2 import v2_bp, require_repo_read, paginate
from endpoints.v2.models_pre_oci import data_model as model
@v2_bp.route('/<repopath:repository>/tags/list', methods=['GET'])
@parse_repository_name()

View file

@ -13,10 +13,10 @@ from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermissi
from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp
from endpoints.v2.errors import InvalidLogin, NameInvalid, InvalidRequest, Unsupported, Unauthorized
from data.interfaces.v2 import pre_oci_model as model
from endpoints.v2.models_pre_oci import data_model as model
from util.cache import no_cache
from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX
from util.security.registry_jwt import (generate_bearer_token, build_context_and_subject, QUAY_TUF_ROOT,
from util.security.registry_jwt import (generate_bearer_token, build_context_and_subject, QUAY_TUF_ROOT,
SIGNER_TUF_ROOT, DISABLED_TUF_ROOT)
logger = logging.getLogger(__name__)
@ -188,7 +188,7 @@ def generate_registry_jwt(auth_result):
def get_tuf_root(repo, namespace, reponame):
if not features.SIGNING or repo is None or not repo.trust_enabled:
return DISABLED_TUF_ROOT
# Users with write access to a repo will see signer-rooted TUF metadata
if ModifyRepositoryPermission(namespace, reponame).can():
return SIGNER_TUF_ROOT