vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix verbs in manifestlist

Browse source 
All registry_tests now pass
This commit is contained in:
Joseph Schorr 2016-09-01 19:00:11 -04:00 • committed by Jimmy Zelinskie
parent 783c9e7a73
commit 3c8b87e086
18 changed files with 517 additions and 247 deletions
Download patch file Download diff file Expand all files Collapse all files

12
data/interfaces/common.py
View file

@ -1,12 +0,0 @@
from image import Repository
from data import model
def repository_for_repo(repo):
""" Returns a Repository object representing the repo data model instance given. """
return Repository(
id=repo.id,
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
)

31
data/interfaces/v1.py
View file

@ -1,8 +1,7 @@
from collections import namedtuple
from app import app, storage as store
from data import model
from data.model import db_transaction
from collections import namedtuple
from util.morecollections import AttrDict
@ -13,19 +12,6 @@ class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'desc
"""
def _repository_for_repo(repo):
"""
Returns a Repository object representing the repo data model instance given.
"""
return Repository(
id=repo.id,
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
)
class DockerRegistryV1DataInterface(object):
"""
Interface that represents all data store interactions required by a Docker Registry v1.
@ -409,12 +395,23 @@ class PreOCIModel(DockerRegistryV1DataInterface):
def change_user_password(cls, user, new_password):
model.user.change_password(user, new_password)
@classmethod
def _repository_for_repo(cls, repo):
""" Returns a Repository object representing the repo data model instance given. """
return Repository(
id=repo.id,
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
)
@classmethod
def get_repository(cls, namespace_name, repo_name):
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return None
return _repository_for_repo(repo)
return cls._repository_for_repo(repo)
@classmethod
def create_repository(cls, namespace_name, repo_name, user=None):
@ -432,4 +429,4 @@ class PreOCIModel(DockerRegistryV1DataInterface):
def get_sorted_matching_repositories(cls, search_term, only_public, can_read, limit):
repos = model.repository.get_sorted_matching_repositories(search_term, only_public, can_read,
limit=limit)
return [_repository_for_repo(repo) for repo in repos]
return [cls._repository_for_repo(repo) for repo in repos]

92
data/interfaces/v2.py
View file

@ -7,10 +7,15 @@ from data import model, database
from data.model import DataModelException
from image.docker.v1 import DockerV1Metadata
_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public'])):
"""
Repository represents a namespaced collection of tags.
"""
class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
"""
ManifestJSON represents a Manifest of any format.
@ -44,47 +49,6 @@ class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'name
"""
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public'])):
"""
Repository represents a namespaced collection of tags.
"""
def _repository_for_repo(repo):
"""
Returns a Repository object representing the repo data model instance given.
"""
return Repository(
id=repo.id,
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
)
def _docker_v1_metadata(namespace_name, repo_name, repo_image):
"""
Returns a DockerV1Metadata object for the given image under the repository with the given
namespace and name. Note that the namespace and name are passed here as an optimization, and are
*not checked* against the image.
"""
return DockerV1Metadata(
namespace_name=namespace_name,
repo_name=repo_name,
image_id=repo_image.docker_image_id,
checksum=repo_image.v1_checksum,
content_checksum=repo_image.storage.content_checksum,
compat_json=repo_image.v1_json_metadata,
created=repo_image.created,
comment=repo_image.comment,
command=repo_image.command,
# TODO: make sure this isn't needed anywhere, as it is expensive to lookup
parent_image_id=None,
)
class DockerRegistryV2DataInterface(object):
"""
Interface that represents all data store interactions required by a Docker Registry v1.
@ -303,12 +267,23 @@ class PreOCIModel(DockerRegistryV2DataInterface):
def repository_is_public(cls, namespace_name, repo_name):
return model.repository.repository_is_public(namespace_name, repo_name)
@classmethod
def _repository_for_repo(cls, repo):
""" Returns a Repository object representing the repo data model instance given. """
return Repository(
id=repo.id,
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
)
@classmethod
def get_repository(cls, namespace_name, repo_name):
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return None
return _repository_for_repo(repo)
return cls._repository_for_repo(repo)
@classmethod
def has_active_tag(cls, namespace_name, repo_name, tag_name):
@ -349,11 +324,32 @@ class PreOCIModel(DockerRegistryV2DataInterface):
tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest)
return [_tag_view(tag) for tag in tags]
@classmethod
def _docker_v1_metadata(cls, namespace_name, repo_name, repo_image):
"""
Returns a DockerV1Metadata object for the given image under the repository with the given
namespace and name. Note that the namespace and name are passed here as an optimization, and are
*not checked* against the image.
"""
return DockerV1Metadata(
namespace_name=namespace_name,
repo_name=repo_name,
image_id=repo_image.docker_image_id,
checksum=repo_image.v1_checksum,
content_checksum=repo_image.storage.content_checksum,
compat_json=repo_image.v1_json_metadata,
created=repo_image.created,
comment=repo_image.comment,
command=repo_image.command,
# TODO: make sure this isn't needed anywhere, as it is expensive to lookup
parent_image_id=None,
)
@classmethod
def get_docker_v1_metadata_by_tag(cls, namespace_name, repo_name, tag_name):
try:
repo_img = model.tag.get_tag_image(namespace_name, repo_name, tag_name, include_storage=True)
return _docker_v1_metadata(namespace_name, repo_name, repo_img)
return cls._docker_v1_metadata(namespace_name, repo_name, repo_img)
except DataModelException:
return None
@ -364,7 +360,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
return {}
images_query = model.image.lookup_repository_images(repo, docker_image_ids)
return {image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image)
return {image.docker_image_id: cls._docker_v1_metadata(namespace_name, repo_name, image)
for image in images_query}
@classmethod
@ -374,7 +370,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
return []
parents = model.image.get_parent_images(namespace_name, repo_name, repo_image)
return [_docker_v1_metadata(namespace_name, repo_name, image) for image in parents]
return [cls._docker_v1_metadata(namespace_name, repo_name, image) for image in parents]
@classmethod
def create_manifest_and_update_tag(cls, namespace_name, repo_name, tag_name, manifest_digest,
@ -406,7 +402,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
repo_image = model.image.synthesize_v1_image(repo, storage_obj, image_id, created, comment,
command, compat_json, parent_image)
return _docker_v1_metadata(repo.namespace_user.username, repo.name, repo_image)
return cls._docker_v1_metadata(repo.namespace_user.username, repo.name, repo_image)
@classmethod
def save_manifest(cls, namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest,
@ -434,7 +430,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
def get_visible_repositories(cls, username, limit, offset):
query = model.repository.get_visible_repositories(username, include_public=(username is None))
query = query.limit(limit).offset(offset)
return [_repository_for_repo(repo) for repo in query]
return [cls._repository_for_repo(repo) for repo in query]
@classmethod
def create_blob_upload(cls, namespace_name, repo_name, upload_uuid, location_name,

304
data/interfaces/verbs.py
View file

@ -1,3 +1,36 @@
from collections import namedtuple
from data import model
from image.docker.v1 import DockerV1Metadata
import json
class DerivedImage(namedtuple('DerivedImage', ['ref', 'blob', 'internal_source_image_db_id'])):
"""
DerivedImage represents a user-facing alias for an image which was derived from another image.
"""
class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
"""
RepositoryReference represents a reference to a Repository, without its full metadata.
"""
class ImageWithBlob(namedtuple('Image', ['image_id', 'blob', 'compat_metadata', 'repository',
'internal_db_id', 'v1_metadata'])):
"""
ImageWithBlob represents a user-facing alias for referencing an image, along with its blob.
"""
class Blob(namedtuple('Blob', ['uuid', 'size', 'uncompressed_size', 'uploading', 'locations'])):
"""
Blob represents an opaque binary blob saved to the storage system.
"""
class TorrentInfo(namedtuple('TorrentInfo', ['piece_length', 'pieces'])):
"""
TorrentInfo represents the torrent piece information associated with a blob.
"""
class VerbsDataInterface(object):
"""
Interface that represents all data store interactions required by the registry's custom HTTP
@ -10,9 +43,280 @@ class VerbsDataInterface(object):
"""
raise NotImplementedError()
@classmethod
def get_manifest_layers_with_blobs(cls, repo_image):
"""
Returns the full set of manifest layers and their associated blobs starting at the given
repository image and working upwards to the root image.
"""
raise NotImplementedError()
@classmethod
def get_blob_path(cls, blob):
"""
Returns the storage path for the given blob.
"""
raise NotImplementedError()
@classmethod
def get_derived_image_signature(cls, derived_image, signer_name):
"""
Returns the signature associated with the derived image and a specific signer or None if none.
"""
raise NotImplementedError()
@classmethod
def set_derived_image_signature(cls, derived_image, signer_name, signature):
"""
Sets the calculated signature for the given derived image and signer to that specified.
"""
raise NotImplementedError()
@classmethod
def delete_derived_image(cls, derived_image):
"""
Deletes a derived image and all of its storage.
"""
raise NotImplementedError()
@classmethod
def set_blob_size(cls, blob, size):
"""
Sets the size field on a blob to the value specified.
"""
raise NotImplementedError()
@classmethod
def get_repo_blob_by_digest(cls, namespace_name, repo_name, digest):
"""
Returns the blob with the given digest under the matching repository or None if none.
"""
raise NotImplementedError()
@classmethod
def get_torrent_info(cls, blob):
"""
Returns the torrent information associated with the given blob or None if none.
"""
raise NotImplementedError()
@classmethod
def set_torrent_info(cls, blob, piece_length, pieces):
"""
Sets the torrent infomation associated with the given blob to that specified.
"""
raise NotImplementedError()
@classmethod
def lookup_derived_image(cls, repo_image, verb, varying_metadata=None):
"""
Looks up the derived image for the given repository image, verb and optional varying metadata
and returns it or None if none.
"""
raise NotImplementedError()
@classmethod
def lookup_or_create_derived_image(cls, repo_image, verb, location, varying_metadata=None):
"""
Looks up the derived image for the given repository image, verb and optional varying metadata
and returns it. If none exists, a new derived image is created.
"""
raise NotImplementedError()
@classmethod
def get_tag_image(cls, namespace_name, repo_name, tag_name):
"""
Returns the image associated with the live tag with the given name under the matching repository
or None if none.
"""
raise NotImplementedError()
class PreOCIModel(VerbsDataInterface):
"""
PreOCIModel implements the data model for the registry's custom HTTP verbs using a database schema
before it was changed to support the OCI specification.
"""
@classmethod
def repository_is_public(cls, namespace_name, repo_name):
return model.repository.repository_is_public(namespace_name, repo_name)
@classmethod
def _docker_v1_metadata(cls, namespace_name, repo_name, repo_image):
"""
Returns a DockerV1Metadata object for the given image under the repository with the given
namespace and name. Note that the namespace and name are passed here as an optimization, and are
*not checked* against the image. Also note that we only fill in the localized data needed by
verbs.
"""
return DockerV1Metadata(
namespace_name=namespace_name,
repo_name=repo_name,
image_id=repo_image.docker_image_id,
checksum=repo_image.v1_checksum,
compat_json=repo_image.v1_json_metadata,
created=repo_image.created,
comment=repo_image.comment,
command=repo_image.command,
# Note: These are not needed in verbs and are expensive to load, so we just skip them.
content_checksum=None,
parent_image_id=None,
)
@classmethod
def get_manifest_layers_with_blobs(cls, repo_image):
repo_image_record = model.image.get_image_by_id(repo_image.repository.namespace_name,
repo_image.repository.name,
repo_image.image_id)
parents = model.image.get_parent_images_with_placements(repo_image.repository.namespace_name,
repo_image.repository.name,
repo_image_record)
yield repo_image
for parent in parents:
metadata = {}
try:
metadata = json.loads(parent.v1_json_metadata)
except ValueError:
pass
yield ImageWithBlob(
image_id=parent.docker_image_id,
blob=cls._blob(parent.storage),
repository=repo_image.repository,
compat_metadata=metadata,
v1_metadata=cls._docker_v1_metadata(repo_image.repository.namespace_name,
repo_image.repository.name, parent),
internal_db_id=parent.id,
)
@classmethod
def get_derived_image_signature(cls, derived_image, signer_name):
storage = model.storage.get_storage_by_uuid(derived_image.blob.uuid)
signature_entry = model.storage.lookup_storage_signature(storage, signer_name)
if signature_entry is None:
return None
return signature_entry.signature
@classmethod
def set_derived_image_signature(cls, derived_image, signer_name, signature):
storage = model.storage.get_storage_by_uuid(derived_image.blob.uuid)
signature_entry = model.storage.find_or_create_storage_signature(storage, signer_name)
signature_entry.signature = signature
signature_entry.uploading = False
signature_entry.save()
@classmethod
def delete_derived_image(cls, derived_image):
model.image.delete_derived_storage_by_uuid(derived_image.blob.uuid)
@classmethod
def set_blob_size(cls, blob, size):
storage_entry = model.storage.get_storage_by_uuid(blob.uuid)
storage_entry.image_size = size
storage_entry.uploading = False
storage_entry.save()
@classmethod
def get_blob_path(cls, blob):
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
return model.storage.get_layer_path(blob_record)
@classmethod
def get_repo_blob_by_digest(cls, namespace_name, repo_name, digest):
try:
blob_record = model.blob.get_repo_blob_by_digest(namespace_name, repo_name, digest)
except model.BlobDoesNotExist:
return None
return cls._blob(blob_record)
@classmethod
def get_torrent_info(cls, blob):
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
try:
torrent_info = model.storage.get_torrent_info(blob_record)
except model.TorrentInfoDoesNotExist:
return None
return TorrentInfo(
pieces=torrent_info.pieces,
piece_length=torrent_info.piece_length,
)
@classmethod
def set_torrent_info(cls, blob, piece_length, pieces):
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
model.storage.save_torrent_info(blob_record, piece_length, pieces)
@classmethod
def lookup_derived_image(cls, repo_image, verb, varying_metadata=None):
blob_record = model.image.find_derived_storage_for_image(repo_image.internal_db_id, verb,
varying_metadata)
if blob_record is None:
return None
return cls._derived_image(blob_record, repo_image)
@classmethod
def _derived_image(cls, blob_record, repo_image):
return DerivedImage(
ref=repo_image.internal_db_id,
blob=cls._blob(blob_record),
internal_source_image_db_id=repo_image.internal_db_id,
)
@classmethod
def _blob(cls, blob_record):
if hasattr(blob_record, 'locations'):
locations = blob_record.locations
else:
locations = model.storage.get_storage_locations(blob_record.uuid)
return Blob(
uuid=blob_record.uuid,
size=blob_record.image_size,
uncompressed_size=blob_record.uncompressed_size,
uploading=blob_record.uploading,
locations=locations,
)
@classmethod
def lookup_or_create_derived_image(cls, repo_image, verb, location, varying_metadata=None):
blob_record = model.image.find_or_create_derived_storage(repo_image.internal_db_id, verb, location,
varying_metadata)
return cls._derived_image(blob_record, repo_image)
@classmethod
def get_tag_image(cls, namespace_name, repo_name, tag_name):
try:
found = model.tag.get_tag_image(namespace_name, repo_name, tag_name, include_storage=True)
except model.DataModelException:
return None
metadata = {}
try:
metadata = json.loads(found.v1_json_metadata)
except ValueError:
pass
return ImageWithBlob(
image_id=found.docker_image_id,
blob=cls._blob(found.storage),
repository=RepositoryReference(
namespace_name=namespace_name,
name=repo_name,
id=found.repository_id,
),
compat_metadata=metadata,
v1_metadata=cls._docker_v1_metadata(namespace_name, repo_name, found),
internal_db_id=found.id,
)