f9e6110f73
Adds support for creating app repos, viewing app repos and seeing the list of app repos in the Quay UI.
542 lines
19 KiB
Python
542 lines
19 KiB
Python
from abc import ABCMeta, abstractmethod
|
|
from collections import namedtuple
|
|
|
|
from namedlist import namedlist
|
|
from peewee import IntegrityError
|
|
from six import add_metaclass
|
|
|
|
from data import model, database
|
|
from data.model import DataModelException
|
|
from image.docker.v1 import DockerV1Metadata
|
|
|
|
_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"
|
|
|
|
|
|
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
|
|
'is_public', 'kind'])):
|
|
"""
|
|
Repository represents a namespaced collection of tags.
|
|
:type id: int
|
|
:type name: string
|
|
:type namespace_name: string
|
|
:type description: string
|
|
:type is_public: bool
|
|
:type kind: string
|
|
"""
|
|
|
|
class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
|
|
"""
|
|
ManifestJSON represents a Manifest of any format.
|
|
"""
|
|
|
|
|
|
class Tag(namedtuple('Tag', ['name', 'repository'])):
|
|
"""
|
|
Tag represents a user-facing alias for referencing a set of Manifests.
|
|
"""
|
|
|
|
|
|
class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count',
|
|
'chunk_count', 'sha_state', 'location_name',
|
|
'storage_metadata', 'piece_sha_state', 'piece_hashes',
|
|
'repo_namespace_name', 'repo_name'])):
|
|
"""
|
|
BlobUpload represents the current state of an Blob being uploaded.
|
|
"""
|
|
|
|
|
|
class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])):
|
|
"""
|
|
Blob represents an opaque binary blob saved to the storage system.
|
|
"""
|
|
|
|
|
|
class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
|
|
"""
|
|
RepositoryReference represents a reference to a Repository, without its full metadata.
|
|
"""
|
|
|
|
class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])):
|
|
"""
|
|
Label represents a key-value pair that describes a particular Manifest.
|
|
"""
|
|
|
|
|
|
@add_metaclass(ABCMeta)
|
|
class DockerRegistryV2DataInterface(object):
|
|
"""
|
|
Interface that represents all data store interactions required by a Docker Registry v1.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def create_repository(self, namespace_name, repo_name, creating_user=None):
|
|
"""
|
|
Creates a new repository under the specified namespace with the given name. The user supplied is
|
|
the user creating the repository, if any.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_repository(self, namespace_name, repo_name):
|
|
"""
|
|
Returns a repository tuple for the repository with the given name under the given namespace.
|
|
Returns None if no such repository was found.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def has_active_tag(self, namespace_name, repo_name, tag_name):
|
|
"""
|
|
Returns whether there is an active tag for the tag with the given name under the matching
|
|
repository, if any, or none if none.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
|
|
"""
|
|
Returns the current manifest for the tag with the given name under the matching repository, if
|
|
any, or None if none.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_manifest_by_digest(self, namespace_name, repo_name, digest):
|
|
"""
|
|
Returns the manifest matching the given digest under the matching repository, if any, or None if
|
|
none.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def delete_manifest_by_digest(self, namespace_name, repo_name, digest):
|
|
"""
|
|
Deletes the manifest with the associated digest (if any) and returns all removed tags that
|
|
pointed to that manifest. If the manifest was not found, returns an empty list.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name):
|
|
"""
|
|
Returns the Docker V1 metadata associated with the tag with the given name under the matching
|
|
repository, if any. If none, returns None.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids):
|
|
"""
|
|
Returns a map of Docker V1 metadata for each given image ID, matched under the repository with
|
|
the given namespace and name. Returns an empty map if the matching repository was not found.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id):
|
|
"""
|
|
Returns an ordered list containing the Docker V1 metadata for each parent of the image with the
|
|
given docker ID under the matching repository. Returns an empty list if the image was not found.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest,
|
|
manifest_bytes):
|
|
"""
|
|
Creates a new manifest with the given digest and byte data, and assigns the tag with the given
|
|
name under the matching repository to it.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def synthesize_v1_image(self, repository, storage, image_id, created, comment, command,
|
|
compat_json, parent_image_id):
|
|
"""
|
|
Synthesizes a V1 image under the specified repository, pointing to the given storage and returns
|
|
the V1 metadata for the synthesized image.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id,
|
|
manifest_digest, manifest_bytes):
|
|
"""
|
|
Saves a manifest pointing to the given leaf image, with the given manifest, under the matching
|
|
repository as a tag with the given name.
|
|
|
|
Returns a boolean whether or not the tag was newly created or not.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def repository_tags(self, namespace_name, repo_name, limit, offset):
|
|
"""
|
|
Returns the active tags under the repository with the given name and namespace.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_visible_repositories(self, username, limit, offset):
|
|
"""
|
|
Returns the repositories visible to the user with the given username, if any.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata):
|
|
"""
|
|
Creates a blob upload under the matching repository with the given UUID and metadata.
|
|
Returns whether the matching repository exists.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid):
|
|
"""
|
|
Searches for a blob upload with the given UUID under the given repository and returns it or None
|
|
if none.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def update_blob_upload(self, blob_upload):
|
|
"""
|
|
Saves any changes to the blob upload object given to the backing data store.
|
|
Fields that can change:
|
|
- uncompressed_byte_count
|
|
- piece_hashes
|
|
- piece_sha_state
|
|
- storage_metadata
|
|
- byte_count
|
|
- chunk_count
|
|
- sha_state
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def delete_blob_upload(self, namespace_name, repo_name, uuid):
|
|
"""
|
|
Deletes the blob upload with the given uuid under the matching repository. If none, does
|
|
nothing.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload,
|
|
expiration_sec):
|
|
"""
|
|
Creates a blob and links a temporary tag with the specified expiration to it under the matching
|
|
repository.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_blob_by_digest(self, namespace_name, repo_name, digest):
|
|
"""
|
|
Returns the blob with the given digest under the matching repository or None if none.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def save_bittorrent_pieces(self, blob, piece_size, piece_bytes):
|
|
"""
|
|
Saves the BitTorrent piece hashes for the given blob.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels):
|
|
"""
|
|
Creates a new labels for the provided manifest.
|
|
"""
|
|
pass
|
|
|
|
|
|
@abstractmethod
|
|
def get_blob_path(self, blob):
|
|
"""
|
|
Once everything is moved over, this could be in util.registry and not even touch the database.
|
|
"""
|
|
pass
|
|
|
|
|
|
class PreOCIModel(DockerRegistryV2DataInterface):
|
|
"""
|
|
PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema
|
|
before it was changed to support the OCI specification.
|
|
"""
|
|
def create_repository(self, namespace_name, repo_name, creating_user=None):
|
|
return model.repository.create_repository(namespace_name, repo_name, creating_user)
|
|
|
|
def get_repository(self, namespace_name, repo_name):
|
|
repo = model.repository.get_repository(namespace_name, repo_name)
|
|
if repo is None:
|
|
return None
|
|
return _repository_for_repo(repo)
|
|
|
|
def has_active_tag(self, namespace_name, repo_name, tag_name):
|
|
try:
|
|
model.tag.get_active_tag(namespace_name, repo_name, tag_name)
|
|
return True
|
|
except database.RepositoryTag.DoesNotExist:
|
|
return False
|
|
|
|
def get_manifest_by_tag(self, namespace_name, repo_name, tag_name):
|
|
try:
|
|
manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name)
|
|
return ManifestJSON(digest=manifest.digest, json=manifest.json_data, media_type=_MEDIA_TYPE)
|
|
except model.InvalidManifestException:
|
|
return None
|
|
|
|
def get_manifest_by_digest(self, namespace_name, repo_name, digest):
|
|
try:
|
|
manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest)
|
|
return ManifestJSON(digest=digest, json=manifest.json_data, media_type=_MEDIA_TYPE)
|
|
except model.InvalidManifestException:
|
|
return None
|
|
|
|
def delete_manifest_by_digest(self, namespace_name, repo_name, digest):
|
|
def _tag_view(tag):
|
|
return Tag(
|
|
name=tag.name,
|
|
repository=RepositoryReference(
|
|
id=tag.repository_id,
|
|
name=repo_name,
|
|
namespace_name=namespace_name,
|
|
)
|
|
)
|
|
|
|
tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest)
|
|
return [_tag_view(tag) for tag in tags]
|
|
|
|
def get_docker_v1_metadata_by_tag(self, namespace_name, repo_name, tag_name):
|
|
try:
|
|
repo_img = model.tag.get_tag_image(namespace_name, repo_name, tag_name, include_storage=True)
|
|
return _docker_v1_metadata(namespace_name, repo_name, repo_img)
|
|
except DataModelException:
|
|
return None
|
|
|
|
def get_docker_v1_metadata_by_image_id(self, namespace_name, repo_name, docker_image_ids):
|
|
repo = model.repository.get_repository(namespace_name, repo_name)
|
|
if repo is None:
|
|
return {}
|
|
|
|
images_query = model.image.lookup_repository_images(repo, docker_image_ids)
|
|
return {image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image)
|
|
for image in images_query}
|
|
|
|
def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id):
|
|
repo_image = model.image.get_repo_image(namespace_name, repo_name, docker_image_id)
|
|
if repo_image is None:
|
|
return []
|
|
|
|
parents = model.image.get_parent_images(namespace_name, repo_name, repo_image)
|
|
return [_docker_v1_metadata(namespace_name, repo_name, image) for image in parents]
|
|
|
|
def create_manifest_and_update_tag(self, namespace_name, repo_name, tag_name, manifest_digest,
|
|
manifest_bytes):
|
|
try:
|
|
model.tag.associate_generated_tag_manifest(namespace_name, repo_name, tag_name,
|
|
manifest_digest, manifest_bytes)
|
|
except IntegrityError:
|
|
# It's already there!
|
|
pass
|
|
|
|
def synthesize_v1_image(self, repository, storage, image_id, created, comment, command,
|
|
compat_json, parent_image_id):
|
|
repo = model.repository.get_repository(repository.namespace_name, repository.name)
|
|
if repo is None:
|
|
raise DataModelException('Unknown repository: %s/%s' % (repository.namespace_name,
|
|
repository.name))
|
|
|
|
parent_image = None
|
|
if parent_image_id is not None:
|
|
parent_image = model.image.get_image(repo, parent_image_id)
|
|
if parent_image is None:
|
|
raise DataModelException('Unknown parent image: %s' % parent_image_id)
|
|
|
|
storage_obj = model.storage.get_storage_by_uuid(storage.uuid)
|
|
if storage_obj is None:
|
|
raise DataModelException('Unknown storage: %s' % storage.uuid)
|
|
|
|
repo_image = model.image.synthesize_v1_image(repo, storage_obj, image_id, created, comment,
|
|
command, compat_json, parent_image)
|
|
return _docker_v1_metadata(repo.namespace_user.username, repo.name, repo_image)
|
|
|
|
def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id,
|
|
manifest_digest, manifest_bytes):
|
|
(_, newly_created) = model.tag.store_tag_manifest(namespace_name, repo_name, tag_name,
|
|
leaf_layer_docker_id, manifest_digest,
|
|
manifest_bytes)
|
|
return newly_created
|
|
|
|
def repository_tags(self, namespace_name, repo_name, limit, offset):
|
|
def _tag_view(tag):
|
|
return Tag(
|
|
name=tag.name,
|
|
repository=RepositoryReference(
|
|
id=tag.repository_id,
|
|
name=repo_name,
|
|
namespace_name=namespace_name,
|
|
)
|
|
)
|
|
|
|
tags_query = model.tag.list_repository_tags(namespace_name, repo_name)
|
|
tags_query = tags_query.limit(limit).offset(offset)
|
|
return [_tag_view(tag) for tag in tags_query]
|
|
|
|
def get_visible_repositories(self, username, limit, offset):
|
|
query = model.repository.get_visible_repositories(username, kind_filter='image',
|
|
include_public=(username is None))
|
|
query = query.limit(limit).offset(offset)
|
|
return [_repository_for_repo(repo) for repo in query]
|
|
|
|
def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata):
|
|
try:
|
|
model.blob.initiate_upload(namespace_name, repo_name, upload_uuid, location_name,
|
|
storage_metadata)
|
|
return True
|
|
except database.Repository.DoesNotExist:
|
|
return False
|
|
|
|
def blob_upload_by_uuid(self, namespace_name, repo_name, upload_uuid):
|
|
try:
|
|
found = model.blob.get_blob_upload(namespace_name, repo_name, upload_uuid)
|
|
except model.InvalidBlobUpload:
|
|
return None
|
|
|
|
return BlobUpload(
|
|
repo_namespace_name=namespace_name,
|
|
repo_name=repo_name,
|
|
uuid=upload_uuid,
|
|
byte_count=found.byte_count,
|
|
uncompressed_byte_count=found.uncompressed_byte_count,
|
|
chunk_count=found.chunk_count,
|
|
sha_state=found.sha_state,
|
|
piece_sha_state=found.piece_sha_state,
|
|
piece_hashes=found.piece_hashes,
|
|
location_name=found.location.name,
|
|
storage_metadata=found.storage_metadata,
|
|
)
|
|
|
|
def update_blob_upload(self, blob_upload):
|
|
# Lookup the blob upload object.
|
|
try:
|
|
blob_upload_record = model.blob.get_blob_upload(blob_upload.repo_namespace_name,
|
|
blob_upload.repo_name, blob_upload.uuid)
|
|
except model.InvalidBlobUpload:
|
|
return
|
|
|
|
blob_upload_record.uncompressed_byte_count = blob_upload.uncompressed_byte_count
|
|
blob_upload_record.piece_hashes = blob_upload.piece_hashes
|
|
blob_upload_record.piece_sha_state = blob_upload.piece_sha_state
|
|
blob_upload_record.storage_metadata = blob_upload.storage_metadata
|
|
blob_upload_record.byte_count = blob_upload.byte_count
|
|
blob_upload_record.chunk_count = blob_upload.chunk_count
|
|
blob_upload_record.sha_state = blob_upload.sha_state
|
|
blob_upload_record.save()
|
|
|
|
def delete_blob_upload(self, namespace_name, repo_name, uuid):
|
|
try:
|
|
found = model.blob.get_blob_upload(namespace_name, repo_name, uuid)
|
|
found.delete_instance()
|
|
except model.InvalidBlobUpload:
|
|
return
|
|
|
|
def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload,
|
|
expiration_sec):
|
|
location_obj = model.storage.get_image_location_for_name(blob_upload.location_name)
|
|
blob_record = model.blob.store_blob_record_and_temp_link(namespace_name, repo_name,
|
|
blob_digest, location_obj.id,
|
|
blob_upload.byte_count,
|
|
expiration_sec,
|
|
blob_upload.uncompressed_byte_count)
|
|
return Blob(
|
|
uuid=blob_record.uuid,
|
|
digest=blob_digest,
|
|
size=blob_upload.byte_count,
|
|
locations=[blob_upload.location_name],
|
|
)
|
|
|
|
def lookup_blobs_by_digest(self, namespace_name, repo_name, digests):
|
|
def _blob_view(blob_record):
|
|
return Blob(
|
|
uuid=blob_record.uuid,
|
|
digest=blob_record.content_checksum,
|
|
size=blob_record.image_size,
|
|
locations=None, # Note: Locations is None in this case.
|
|
)
|
|
|
|
repo = model.repository.get_repository(namespace_name, repo_name)
|
|
if repo is None:
|
|
return {}
|
|
query = model.storage.lookup_repo_storages_by_content_checksum(repo, digests)
|
|
return {storage.content_checksum: _blob_view(storage) for storage in query}
|
|
|
|
def get_blob_by_digest(self, namespace_name, repo_name, digest):
|
|
try:
|
|
blob_record = model.blob.get_repo_blob_by_digest(namespace_name, repo_name, digest)
|
|
return Blob(
|
|
uuid=blob_record.uuid,
|
|
digest=digest,
|
|
size=blob_record.image_size,
|
|
locations=blob_record.locations,
|
|
)
|
|
except model.BlobDoesNotExist:
|
|
return None
|
|
|
|
def save_bittorrent_pieces(self, blob, piece_size, piece_bytes):
|
|
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
|
|
model.storage.save_torrent_info(blob_record, piece_size, piece_bytes)
|
|
|
|
def create_manifest_labels(self, namespace_name, repo_name, manifest_digest, labels):
|
|
if not labels:
|
|
# No point in doing anything more.
|
|
return
|
|
|
|
tag_manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, manifest_digest)
|
|
for label in labels:
|
|
model.label.create_manifest_label(tag_manifest, label.key, label.value, label.source_type,
|
|
label.media_type)
|
|
|
|
def get_blob_path(self, blob):
|
|
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
|
|
return model.storage.get_layer_path(blob_record)
|
|
|
|
|
|
def _docker_v1_metadata(namespace_name, repo_name, repo_image):
|
|
"""
|
|
Returns a DockerV1Metadata object for the given Pre-OCI repo_image under the
|
|
repository with the given namespace and name. Note that the namespace and
|
|
name are passed here as an optimization, and are *not checked* against the
|
|
image.
|
|
"""
|
|
return DockerV1Metadata(
|
|
namespace_name=namespace_name,
|
|
repo_name=repo_name,
|
|
image_id=repo_image.docker_image_id,
|
|
checksum=repo_image.v1_checksum,
|
|
content_checksum=repo_image.storage.content_checksum,
|
|
compat_json=repo_image.v1_json_metadata,
|
|
created=repo_image.created,
|
|
comment=repo_image.comment,
|
|
command=repo_image.command,
|
|
# TODO: make sure this isn't needed anywhere, as it is expensive to lookup
|
|
parent_image_id=None,
|
|
)
|
|
|
|
|
|
def _repository_for_repo(repo):
|
|
""" Returns a Repository object representing the Pre-OCI data model repo instance given. """
|
|
return Repository(
|
|
id=repo.id,
|
|
name=repo.name,
|
|
namespace_name=repo.namespace_user.username,
|
|
description=repo.description,
|
|
is_public=model.repository.is_repository_public(repo),
|
|
kind=model.repository.get_repo_kind_name(repo),
|
|
)
|
|
|
|
|
|
pre_oci_model = PreOCIModel()
|