endpoints.v2: new fs layout for data interface

Fixes QUAY-658

endpoints/v2/__init__.py

@@ -15,9 +15,9 @@ from auth.auth_context import get_grant_context
 from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
                               AdministerRepositoryPermission)
 from auth.registry_jwt_auth import process_registry_jwt_auth, get_auth_headers
-from data.interfaces.v2 import pre_oci_model as model
 from endpoints.decorators import anon_protect, anon_allowed
 from endpoints.v2.errors import V2RegistryException, Unauthorized, Unsupported, NameUnknown
+from endpoints.v2.models_pre_oci import data_model as model
 from util.http import abort
 from util.metrics.metricqueue import time_blueprint
 from util.registry.dockerver import docker_version

endpoints/v2/blob.py

@@ -10,13 +10,13 @@ import resumablehashlib
 from app import storage, app, get_app_url, metric_queue
 from auth.registry_jwt_auth import process_registry_jwt_auth
 from data import database
-from data.interfaces.v2 import pre_oci_model as model
 from digest import digest_tools
 from endpoints.common import parse_repository_name
+from endpoints.decorators import anon_protect
 from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream
 from endpoints.v2.errors import (BlobUnknown, BlobUploadInvalid, BlobUploadUnknown, Unsupported,
                                  NameUnknown, LayerTooLarge)
-from endpoints.decorators import anon_protect
+from endpoints.v2.models_pre_oci import data_model as model
 from util.cache import cache_control
 from util.registry.filelike import wrap_with_handler, StreamSlice
 from util.registry.gzipstream import calculate_size_handler

endpoints/v2/catalog.py

@@ -5,7 +5,7 @@ from flask import jsonify
 from auth.registry_jwt_auth import process_registry_jwt_auth, get_granted_entity
 from endpoints.decorators import anon_protect
 from endpoints.v2 import v2_bp, paginate
-from data.interfaces.v2 import pre_oci_model as model
+from endpoints.v2.models_pre_oci import data_model as model
 
 @v2_bp.route('/_catalog', methods=['GET'])
 @process_registry_jwt_auth()

endpoints/v2/manifest.py

@@ -8,14 +8,15 @@ import features
 
 from app import docker_v2_signing_key, app, metric_queue
 from auth.registry_jwt_auth import process_registry_jwt_auth
-from data.interfaces.v2 import pre_oci_model as model, Label
 from digest import digest_tools
 from endpoints.common import parse_repository_name
 from endpoints.decorators import anon_protect
-from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
-from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
-                                 NameInvalid)
 from endpoints.notificationhelper import spawn_notification
+from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
+from endpoints.v2.errors import (
+  BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid)
+from endpoints.v2.models_interface import Label
+from endpoints.v2.models_pre_oci import data_model as model
 from image.docker import ManifestException
 from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
 from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES

endpoints/v2/models_interface.py

@@ -0,0 +1,255 @@
+from abc import ABCMeta, abstractmethod
+from collections import namedtuple
+
+from namedlist import namedlist
+from six import add_metaclass
+
+
+class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
+                                           'is_public', 'kind', 'trust_enabled'])):
+  """
+  Repository represents a namespaced collection of tags.
+  :type id: int
+  :type name: string
+  :type namespace_name: string
+  :type description: string
+  :type is_public: bool
+  :type kind: string
+  :type trust_enabled: bool
+  """
+
+class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
+  """
+  ManifestJSON represents a Manifest of any format.
+  """
+
+
+class Tag(namedtuple('Tag', ['name', 'repository'])):
+  """
+  Tag represents a user-facing alias for referencing a set of Manifests.
+  """
+
+
+class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count',
+                                          'chunk_count', 'sha_state', 'location_name',
+                                          'storage_metadata', 'piece_sha_state', 'piece_hashes',
+                                          'repo_namespace_name', 'repo_name'])):
+  """
+  BlobUpload represents the current state of an Blob being uploaded.
+  """
+
+
+class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])):
+  """
+  Blob represents an opaque binary blob saved to the storage system.
+  """
+
+
+class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
+  """
+  RepositoryReference represents a reference to a Repository, without its full metadata.
+  """
+
+class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])):
+  """
+  Label represents a key-value pair that describes a particular Manifest.
+  """
+
+
+@add_metaclass(ABCMeta)
+class DockerRegistryV2DataInterface(object):
+  """
+  Interface that represents all data store interactions required by a Docker Registry v1.
+  """
+
+  @abstractmethod
+  def create_repository(self, namespace_name, repo_name, creating_user=None):
+    """
+    Creates a new repository under the specified namespace with the given name. The user supplied is
+    the user creating the repository, if any.
+    """
+    pass
+
+  @abstractmethod
+  def get_repository(self, namespace_name, repo_name):
+    """
+    Returns a repository tuple for the repository with the given name under the given namespace.
+    Returns None if no such repository was found.
+    """
+    pass
+
+  @abstractmethod
+  def has_active_tag(self, namespace_name, repo_name, tag_name):
+    """
+    Returns whether there is an active tag for the tag with the given name under the matching
+    repository, if any, or none if none.
+    """
+    pass
+
+  @abstractmethod
+  def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
+    """
+    Returns the current manifest for the tag with the given name under the matching repository, if
+    any, or None if none.
+    """
+    pass
+
+  @abstractmethod
+  def get_manifest_by_digest(self, namespace_name, repo_name, digest):
+    """
+    Returns the manifest matching the given digest under the matching repository, if any, or None if
+    none.
+    """
+    pass
+
+  @abstractmethod
+  def delete_manifest_by_digest(self, namespace_name, repo_name, digest):
+    """
+    Deletes the manifest with the associated digest (if any) and returns all removed tags that
+    pointed to that manifest. If the manifest was not found, returns an empty list.
+    """
+    pass
+
+  @abstractmethod
+  def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name):
+    """
+    Returns the Docker V1 metadata associated with the tag with the given name under the matching
+    repository, if any. If none, returns None.
+    """
+    pass
+
+  @abstractmethod
+  def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids):
+    """
+    Returns a map of Docker V1 metadata for each given image ID, matched under the repository with
+    the given namespace and name. Returns an empty map if the matching repository was not found.
+    """
+    pass
+
+  @abstractmethod
+  def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id):
+    """
+    Returns an ordered list containing the Docker V1 metadata for each parent of the image with the
+    given docker ID under the matching repository. Returns an empty list if the image was not found.
+    """
+    pass
+
+  @abstractmethod
+  def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest,
+                                     manifest_bytes):
+    """
+    Creates a new manifest with the given digest and byte data, and assigns the tag with the given
+    name under the matching repository to it.
+    """
+    pass
+
+  @abstractmethod
+  def synthesize_v1_image(self, repository, storage, image_id, created, comment, command,
+                          compat_json, parent_image_id):
+    """
+    Synthesizes a V1 image under the specified repository, pointing to the given storage and returns
+    the V1 metadata for the synthesized image.
+    """
+    pass
+
+  @abstractmethod
+  def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id,
+                    manifest_digest, manifest_bytes):
+    """
+    Saves a manifest pointing to the given leaf image, with the given manifest, under the matching
+    repository as a tag with the given name.
+
+    Returns a boolean whether or not the tag was newly created or not.
+    """
+    pass
+
+  @abstractmethod
+  def repository_tags(self, namespace_name, repo_name, limit, offset):
+    """
+    Returns the active tags under the repository with the given name and namespace.
+    """
+    pass
+
+  @abstractmethod
+  def get_visible_repositories(self, username, limit, offset):
+    """
+    Returns the repositories visible to the user with the given username, if any.
+    """
+    pass
+
+  @abstractmethod
+  def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata):
+    """
+    Creates a blob upload under the matching repository with the given UUID and metadata.
+    Returns whether the matching repository exists.
+    """
+    pass
+
+  @abstractmethod
+  def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid):
+    """
+    Searches for a blob upload with the given UUID under the given repository and returns it or None
+    if none.
+    """
+    pass
+
+  @abstractmethod
+  def update_blob_upload(self, blob_upload):
+    """
+    Saves any changes to the blob upload object given to the backing data store.
+    Fields that can change:
+      - uncompressed_byte_count
+      - piece_hashes
+      - piece_sha_state
+      - storage_metadata
+      - byte_count
+      - chunk_count
+      - sha_state
+    """
+    pass
+
+  @abstractmethod
+  def delete_blob_upload(self, namespace_name, repo_name, uuid):
+    """
+    Deletes the blob upload with the given uuid under the matching repository. If none, does
+    nothing.
+    """
+    pass
+
+  @abstractmethod
+  def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload,
+                               expiration_sec):
+    """
+    Creates a blob and links a temporary tag with the specified expiration to it under the matching
+    repository.
+    """
+    pass
+
+  @abstractmethod
+  def get_blob_by_digest(self, namespace_name, repo_name, digest):
+    """
+    Returns the blob with the given digest under the matching repository or None if none.
+    """
+    pass
+
+  @abstractmethod
+  def save_bittorrent_pieces(self, blob, piece_size, piece_bytes):
+    """
+    Saves the BitTorrent piece hashes for the given blob.
+    """
+    pass
+
+  @abstractmethod
+  def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels):
+    """
+    Creates a new labels for the provided manifest.
+    """
+    pass
+
+
+  @abstractmethod
+  def get_blob_path(self, blob):
+    """
+    Once everything is moved over, this could be in util.registry and not even touch the database.
+    """
+    pass

endpoints/v2/models_pre_oci.py

@@ -1,267 +1,22 @@
-from abc import ABCMeta, abstractmethod
-from collections import namedtuple
-
-from namedlist import namedlist
 from peewee import IntegrityError
-from six import add_metaclass
 
 from data import model, database
 from data.model import DataModelException
+from endpoints.v2.models_interface import (
+  Blob,
+  BlobUpload,
+  DockerRegistryV2DataInterface,
+  ManifestJSON,
+  Repository,
+  RepositoryReference,
+  Tag,
+)
 from image.docker.v1 import DockerV1Metadata
 
+
 _MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"
 
 
-class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
-                                           'is_public', 'kind', 'trust_enabled'])):
-  """
-  Repository represents a namespaced collection of tags.
-  :type id: int
-  :type name: string
-  :type namespace_name: string
-  :type description: string
-  :type is_public: bool
-  :type kind: string
-  :type trust_enabled: bool
-  """
-
-class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
-  """
-  ManifestJSON represents a Manifest of any format.
-  """
-
-
-class Tag(namedtuple('Tag', ['name', 'repository'])):
-  """
-  Tag represents a user-facing alias for referencing a set of Manifests.
-  """
-
-
-class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count',
-                                          'chunk_count', 'sha_state', 'location_name',
-                                          'storage_metadata', 'piece_sha_state', 'piece_hashes',
-                                          'repo_namespace_name', 'repo_name'])):
-  """
-  BlobUpload represents the current state of an Blob being uploaded.
-  """
-
-
-class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])):
-  """
-  Blob represents an opaque binary blob saved to the storage system.
-  """
-
-
-class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
-  """
-  RepositoryReference represents a reference to a Repository, without its full metadata.
-  """
-
-class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])):
-  """
-  Label represents a key-value pair that describes a particular Manifest.
-  """
-
-
-@add_metaclass(ABCMeta)
-class DockerRegistryV2DataInterface(object):
-  """
-  Interface that represents all data store interactions required by a Docker Registry v1.
-  """
-
-  @abstractmethod
-  def create_repository(self, namespace_name, repo_name, creating_user=None):
-    """
-    Creates a new repository under the specified namespace with the given name. The user supplied is
-    the user creating the repository, if any.
-    """
-    pass
-
-  @abstractmethod
-  def get_repository(self, namespace_name, repo_name):
-    """
-    Returns a repository tuple for the repository with the given name under the given namespace.
-    Returns None if no such repository was found.
-    """
-    pass
-
-  @abstractmethod
-  def has_active_tag(self, namespace_name, repo_name, tag_name):
-    """
-    Returns whether there is an active tag for the tag with the given name under the matching
-    repository, if any, or none if none.
-    """
-    pass
-
-  @abstractmethod
-  def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
-    """
-    Returns the current manifest for the tag with the given name under the matching repository, if
-    any, or None if none.
-    """
-    pass
-
-  @abstractmethod
-  def get_manifest_by_digest(self, namespace_name, repo_name, digest):
-    """
-    Returns the manifest matching the given digest under the matching repository, if any, or None if
-    none.
-    """
-    pass
-
-  @abstractmethod
-  def delete_manifest_by_digest(self, namespace_name, repo_name, digest):
-    """
-    Deletes the manifest with the associated digest (if any) and returns all removed tags that
-    pointed to that manifest. If the manifest was not found, returns an empty list.
-    """
-    pass
-
-  @abstractmethod
-  def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name):
-    """
-    Returns the Docker V1 metadata associated with the tag with the given name under the matching
-    repository, if any. If none, returns None.
-    """
-    pass
-
-  @abstractmethod
-  def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids):
-    """
-    Returns a map of Docker V1 metadata for each given image ID, matched under the repository with
-    the given namespace and name. Returns an empty map if the matching repository was not found.
-    """
-    pass
-
-  @abstractmethod
-  def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id):
-    """
-    Returns an ordered list containing the Docker V1 metadata for each parent of the image with the
-    given docker ID under the matching repository. Returns an empty list if the image was not found.
-    """
-    pass
-
-  @abstractmethod
-  def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest,
-                                     manifest_bytes):
-    """
-    Creates a new manifest with the given digest and byte data, and assigns the tag with the given
-    name under the matching repository to it.
-    """
-    pass
-
-  @abstractmethod
-  def synthesize_v1_image(self, repository, storage, image_id, created, comment, command,
-                          compat_json, parent_image_id):
-    """
-    Synthesizes a V1 image under the specified repository, pointing to the given storage and returns
-    the V1 metadata for the synthesized image.
-    """
-    pass
-
-  @abstractmethod
-  def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id,
-                    manifest_digest, manifest_bytes):
-    """
-    Saves a manifest pointing to the given leaf image, with the given manifest, under the matching
-    repository as a tag with the given name.
-
-    Returns a boolean whether or not the tag was newly created or not.
-    """
-    pass
-
-  @abstractmethod
-  def repository_tags(self, namespace_name, repo_name, limit, offset):
-    """
-    Returns the active tags under the repository with the given name and namespace.
-    """
-    pass
-
-  @abstractmethod
-  def get_visible_repositories(self, username, limit, offset):
-    """
-    Returns the repositories visible to the user with the given username, if any.
-    """
-    pass
-
-  @abstractmethod
-  def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata):
-    """
-    Creates a blob upload under the matching repository with the given UUID and metadata.
-    Returns whether the matching repository exists.
-    """
-    pass
-
-  @abstractmethod
-  def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid):
-    """
-    Searches for a blob upload with the given UUID under the given repository and returns it or None
-    if none.
-    """
-    pass
-
-  @abstractmethod
-  def update_blob_upload(self, blob_upload):
-    """
-    Saves any changes to the blob upload object given to the backing data store.
-    Fields that can change:
-      - uncompressed_byte_count
-      - piece_hashes
-      - piece_sha_state
-      - storage_metadata
-      - byte_count
-      - chunk_count
-      - sha_state
-    """
-    pass
-
-  @abstractmethod
-  def delete_blob_upload(self, namespace_name, repo_name, uuid):
-    """
-    Deletes the blob upload with the given uuid under the matching repository. If none, does
-    nothing.
-    """
-    pass
-
-  @abstractmethod
-  def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload,
-                               expiration_sec):
-    """
-    Creates a blob and links a temporary tag with the specified expiration to it under the matching
-    repository.
-    """
-    pass
-
-  @abstractmethod
-  def get_blob_by_digest(self, namespace_name, repo_name, digest):
-    """
-    Returns the blob with the given digest under the matching repository or None if none.
-    """
-    pass
-
-  @abstractmethod
-  def save_bittorrent_pieces(self, blob, piece_size, piece_bytes):
-    """
-    Saves the BitTorrent piece hashes for the given blob.
-    """
-    pass
-
-  @abstractmethod
-  def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels):
-    """
-    Creates a new labels for the provided manifest.
-    """
-    pass
-
-
-  @abstractmethod
-  def get_blob_path(self, blob):
-    """
-    Once everything is moved over, this could be in util.registry and not even touch the database.
-    """
-    pass
-
-
 class PreOCIModel(DockerRegistryV2DataInterface):
   """
   PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema
@@ -544,4 +299,4 @@ def _repository_for_repo(repo):
   )
 
 
-pre_oci_model = PreOCIModel()
+data_model = PreOCIModel()

endpoints/v2/tag.py

@@ -2,9 +2,9 @@ from flask import jsonify
 
 from auth.registry_jwt_auth import process_registry_jwt_auth
 from endpoints.common import parse_repository_name
-from endpoints.v2 import v2_bp, require_repo_read, paginate
 from endpoints.decorators import anon_protect
-from data.interfaces.v2 import pre_oci_model as model
+from endpoints.v2 import v2_bp, require_repo_read, paginate
+from endpoints.v2.models_pre_oci import data_model as model
 
 @v2_bp.route('/<repopath:repository>/tags/list', methods=['GET'])
 @parse_repository_name()

endpoints/v2/v2auth.py

@@ -13,7 +13,7 @@ from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermissi
 from endpoints.decorators import anon_protect
 from endpoints.v2 import v2_bp
 from endpoints.v2.errors import InvalidLogin, NameInvalid, InvalidRequest, Unsupported, Unauthorized
-from data.interfaces.v2 import pre_oci_model as model
+from endpoints.v2.models_pre_oci import data_model as model
 from util.cache import no_cache
 from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX
 from util.security.registry_jwt import (generate_bearer_token, build_context_and_subject, QUAY_TUF_ROOT,