vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

create interfaces for v1 and v2 data model

Browse source
This commit is contained in:
Jimmy Zelinskie 2016-08-30 15:05:15 -04:00
parent b775458d4b
commit c06d395f96
14 changed files with 1048 additions and 732 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

620
data/interfaces/v1.py
View file

@ -1,263 +1,435 @@
from collections import namedtuple
from app import app, storage as store from app import app, storage as store
from data import model from data import model
from data.model import db_transaction from data.model import db_transaction
from util.morecollections import AttrDict from util.morecollections import AttrDict
from data.interfaces.common import repository_for_repo
def placement_locations_docker_v1(namespace_name, repo_name, image_id):
""" Returns all the placements for the image with the given V1 Docker ID, found under the class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
given repository or None if no image was found. 'is_public'])):
""" """
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) Repository represents a namespaced collection of tags.
if repo_image is None or repo_image.storage is None:
return None
return repo_image.storage.locations
def placement_locations_and_path_docker_v1(namespace_name, repo_name, image_id):
""" Returns a tuple of the placements and storage path location for the image with the
given V1 Docker ID, found under the given repository or None if no image was found.
""" """
repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id)
if not repo_image or repo_image.storage is None:
return None, None
return repo_image.storage.locations, model.storage.get_layer_path(repo_image.storage)
def docker_v1_metadata(namespace_name, repo_name, image_id): def _repository_for_repo(repo):
""" Returns various pieces of metadata associated with an image with the given V1 Docker ID,
including the checksum and its V1 JSON metadata.
""" """
repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) Returns a Repository object representing the repo data model instance given.
if repo_image is None:
return None
return AttrDict({
'namespace_name': namespace_name,
'repo_name': repo_name,
'image_id': image_id,
'checksum': repo_image.v1_checksum,
'compat_json': repo_image.v1_json_metadata,
})
def update_docker_v1_metadata(namespace_name, repo_name, image_id, created_date_str, comment,
command, compat_json, parent_image_id=None):
""" Updates various pieces of V1 metadata associated with a particular image. """
parent_image = None
if parent_image_id is not None:
parent_image = model.image.get_repo_image(namespace_name, repo_name, parent_image_id)
model.image.set_image_metadata(image_id, namespace_name, repo_name, created_date_str, comment,
command, compat_json, parent=parent_image)
def storage_exists(namespace_name, repo_name, image_id):
""" Returns whether storage already exists for the image with the V1 Docker ID under the
given repository.
""" """
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) return Repository(
if repo_image is None or repo_image.storage is None: id=repo.id,
return False name=repo.name,
namespace_name=repo.namespace_user.username,
if repo_image.storage.uploading: description=repo.description,
return False is_public=model.repository.is_repository_public(repo)
)
layer_path = model.storage.get_layer_path(repo_image.storage)
return store.exists(repo_image.storage.locations, layer_path)
def store_docker_v1_checksums(namespace_name, repo_name, image_id, checksum, content_checksum): class DockerRegistryV1DataInterface(object):
""" Stores the various V1 checksums for the image with the V1 Docker ID. """ """
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) Interface that represents all data store interactions required by a Docker Registry v1.
if repo_image is None or repo_image.storage is None: """
return
with db_transaction(): @classmethod
repo_image.storage.content_checksum = content_checksum def placement_locations_docker_v1(cls, namespace_name, repo_name, image_id):
repo_image.v1_checksum = checksum """
Returns all the placements for the image with the given V1 Docker ID, found under the given
repository or None if no image was found.
"""
raise NotImplementedError()
@classmethod
def placement_locations_and_path_docker_v1(cls, namespace_name, repo_name, image_id):
"""
Returns all the placements for the image with the given V1 Docker ID, found under the given
repository or None if no image was found.
"""
raise NotImplementedError()
@classmethod
def docker_v1_metadata(cls, namespace_name, repo_name, image_id):
"""
Returns various pieces of metadata associated with an image with the given V1 Docker ID,
including the checksum and its V1 JSON metadata.
"""
raise NotImplementedError()
@classmethod
def update_docker_v1_metadata(cls, namespace_name, repo_name, image_id, created_date_str, comment,
command, compat_json, parent_image_id=None):
"""
Updates various pieces of V1 metadata associated with a particular image.
"""
raise NotImplementedError()
@classmethod
def storage_exists(cls, namespace_name, repo_name, image_id):
"""
Returns whether storage already exists for the image with the V1 Docker ID under the given
repository.
"""
raise NotImplementedError()
@classmethod
def store_docker_v1_checksums(cls, namespace_name, repo_name, image_id, checksum, content_checksum):
"""
Stores the various V1 checksums for the image with the V1 Docker ID.
"""
raise NotImplementedError()
@classmethod
def is_image_uploading(cls, namespace_name, repo_name, image_id):
"""
Returns whether the image with the V1 Docker ID is currently marked as uploading.
"""
raise NotImplementedError()
@classmethod
def update_image_uploading(cls, namespace_name, repo_name, image_id, is_uploading):
""" Marks the image with the V1 Docker ID with the given uploading status. """
raise NotImplementedError()
@classmethod
def update_image_sizes(cls, namespace_name, repo_name, image_id, size, uncompressed_size):
"""
Updates the sizing information for the image with the given V1 Docker ID.
"""
raise NotImplementedError()
@classmethod
def get_image_size(cls, namespace_name, repo_name, image_id):
"""
Returns the wire size of the image with the given Docker V1 ID.
"""
raise NotImplementedError()
@classmethod
def create_bittorrent_pieces(cls, namespace_name, repo_name, image_id, pieces_bytes):
"""
Saves the BitTorrent piece hashes for the image with the given Docker V1 ID.
"""
raise NotImplementedError()
@classmethod
def image_ancestry(cls, namespace_name, repo_name, image_id):
"""
Returns a list containing the full ancestry of Docker V1 IDs, in order, for the image with the
given Docker V1 ID.
"""
raise NotImplementedError()
@classmethod
def repository_exists(cls, namespace_name, repo_name):
"""
Returns whether the repository with the given name and namespace exists.
"""
raise NotImplementedError()
@classmethod
def create_or_link_image(cls, username, namespace_name, repo_name, image_id, storage_location):
"""
Adds the given image to the given repository, by either linking to an existing image visible to
the user with the given username, or creating a new one if no existing image matches.
"""
raise NotImplementedError()
@classmethod
def create_temp_hidden_tag(cls, namespace_name, repo_name, image_id, expiration):
"""
Creates a hidden tag under the matching namespace pointing to the image with the given V1 Docker
ID.
"""
raise NotImplementedError()
@classmethod
def list_tags(cls, namespace_name, repo_name):
"""
Returns all the tags defined in the repository with the given namespace and name.
"""
raise NotImplementedError()
@classmethod
def create_or_update_tag(cls, namespace_name, repo_name, image_id, tag_name):
"""
Creates or updates a tag under the matching repository to point to the image with the given
Docker V1 ID.
"""
raise NotImplementedError()
@classmethod
def find_image_id_by_tag(cls, namespace_name, repo_name, tag_name):
"""
Returns the Docker V1 image ID for the HEAD image for the tag with the given name under the
matching repository, or None if none.
"""
raise NotImplementedError()
@classmethod
def delete_tag(cls, namespace_name, repo_name, tag_name):
""" Deletes the given tag from the given repository. """
raise NotImplementedError()
@classmethod
def load_token(cls, token):
"""
Loads the data associated with the given (deprecated) access token, and, if
found returns True.
"""
raise NotImplementedError()
@classmethod
def verify_robot(cls, username, token):
"""
Returns True if the given robot username and token match an existing robot
account.
"""
raise NotImplementedError()
@classmethod
def change_user_password(cls, user, new_password):
"""
Changes the password associated with the given user.
"""
raise NotImplementedError()
@classmethod
def get_repository(cls, namespace_name, repo_name):
"""
Returns the repository with the given name under the given namespace or None
if none.
"""
raise NotImplementedError()
@classmethod
def create_repository(cls, namespace_name, repo_name, user=None):
"""
Creates a new repository under the given namespace with the given name, for
the given user.
"""
raise NotImplementedError()
@classmethod
def repository_is_public(cls, namespace_name, repo_name):
"""
Returns whether the repository with the given name under the given namespace
is public. If no matching repository was found, returns False.
"""
raise NotImplementedError()
@classmethod
def validate_oauth_token(cls, token):
""" Returns whether the given OAuth token validates. """
raise NotImplementedError()
@classmethod
def get_sorted_matching_repositories(cls, search_term, only_public, can_read, limit):
"""
Returns a sorted list of repositories matching the given search term.
can_read is a callback that will be invoked for each repository found, to
filter results to only those visible to the current user (if any).
"""
raise NotImplementedError()
class PreOCIModel(DockerRegistryV1DataInterface):
"""
PreOCIModel implements the data model for the v1 Docker Registry protocol using a database schema
before it was changed to support the OCI specification.
"""
@classmethod
def placement_locations_docker_v1(cls, namespace_name, repo_name, image_id):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None:
return None
return repo_image.storage.locations
@classmethod
def placement_locations_and_path_docker_v1(cls, namespace_name, repo_name, image_id):
repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id)
if not repo_image or repo_image.storage is None:
return None, None
return repo_image.storage.locations, model.storage.get_layer_path(repo_image.storage)
@classmethod
def docker_v1_metadata(cls, namespace_name, repo_name, image_id):
repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id)
if repo_image is None:
return None
return AttrDict({
'namespace_name': namespace_name,
'repo_name': repo_name,
'image_id': image_id,
'checksum': repo_image.v1_checksum,
'compat_json': repo_image.v1_json_metadata,
})
@classmethod
def update_docker_v1_metadata(cls, namespace_name, repo_name, image_id, created_date_str, comment,
command, compat_json, parent_image_id=None):
parent_image = None
if parent_image_id is not None:
parent_image = model.image.get_repo_image(namespace_name, repo_name, parent_image_id)
model.image.set_image_metadata(image_id, namespace_name, repo_name, created_date_str, comment,
command, compat_json, parent=parent_image)
@classmethod
def storage_exists(cls, namespace_name, repo_name, image_id):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None:
return False
if repo_image.storage.uploading:
return False
layer_path = model.storage.get_layer_path(repo_image.storage)
return store.exists(repo_image.storage.locations, layer_path)
@classmethod
def store_docker_v1_checksums(cls, namespace_name, repo_name, image_id, checksum, content_checksum):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None:
return
with db_transaction():
repo_image.storage.content_checksum = content_checksum
repo_image.v1_checksum = checksum
repo_image.storage.save()
repo_image.save()
@classmethod
def is_image_uploading(cls, namespace_name, repo_name, image_id):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None:
return False
return repo_image.storage.uploading
@classmethod
def update_image_uploading(cls, namespace_name, repo_name, image_id, is_uploading):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None:
return
repo_image.storage.uploading = is_uploading
repo_image.storage.save() repo_image.storage.save()
repo_image.save() return repo_image.storage
@classmethod
def update_image_sizes(cls, namespace_name, repo_name, image_id, size, uncompressed_size):
model.storage.set_image_storage_metadata(image_id, namespace_name, repo_name, size,
uncompressed_size)
def is_image_uploading(namespace_name, repo_name, image_id): @classmethod
""" Returns whether the image with the V1 Docker ID is currently marked as uploading. """ def get_image_size(cls, namespace_name, repo_name, image_id):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None: if repo_image is None or repo_image.storage is None:
return False return None
return repo_image.storage.image_size
return repo_image.storage.uploading @classmethod
def create_bittorrent_pieces(cls, namespace_name, repo_name, image_id, pieces_bytes):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None:
return
model.storage.save_torrent_info(repo_image.storage, app.config['BITTORRENT_PIECE_SIZE'],
pieces_bytes)
def update_image_uploading(namespace_name, repo_name, image_id, is_uploading): @classmethod
""" Marks the image with the V1 Docker ID with the given uploading status. """ def image_ancestry(cls, namespace_name, repo_name, image_id):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) try:
if repo_image is None or repo_image.storage is None: image = model.image.get_image_by_id(namespace_name, repo_name, image_id)
return except model.InvalidImageException:
return None
repo_image.storage.uploading = is_uploading parents = model.image.get_parent_images(namespace_name, repo_name, image)
repo_image.storage.save() ancestry_docker_ids = [image.docker_image_id]
return repo_image.storage ancestry_docker_ids.extend([parent.docker_image_id for parent in parents])
return ancestry_docker_ids
@classmethod
def repository_exists(cls, namespace_name, repo_name):
repo = model.repository.get_repository(namespace_name, repo_name)
return repo is not None
def update_image_sizes(namespace_name, repo_name, image_id, size, uncompressed_size): @classmethod
""" Updates the sizing information for the image with the given V1 Docker ID. """ def create_or_link_image(cls, username, namespace_name, repo_name, image_id, storage_location):
model.storage.set_image_storage_metadata(image_id, namespace_name, repo_name, size, repo = model.repository.get_repository(namespace_name, repo_name)
uncompressed_size) model.image.find_create_or_link_image(image_id, repo, username, {}, storage_location)
@classmethod
def create_temp_hidden_tag(cls, namespace_name, repo_name, image_id, expiration):
repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id)
if repo_image is None:
return
def get_image_size(namespace_name, repo_name, image_id): repo = repo_image.repository
""" Returns the wire size of the image with the given Docker V1 ID. """ model.tag.create_temporary_hidden_tag(repo, repo_image, expiration)
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
if repo_image is None or repo_image.storage is None:
return None
return repo_image.storage.image_size @classmethod
def list_tags(cls, namespace_name, repo_name):
return model.tag.list_repository_tags(namespace_name, repo_name)
@classmethod
def create_or_update_tag(cls, namespace_name, repo_name, image_id, tag_name):
model.tag.create_or_update_tag(namespace_name, repo_name, tag_name, image_id)
def create_bittorrent_pieces(namespace_name, repo_name, image_id, pieces_bytes): @classmethod
""" Saves the bittorrent piece hashes for the image with the given Docker V1 ID. """ def find_image_id_by_tag(cls, namespace_name, repo_name, tag_name):
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id) try:
if repo_image is None or repo_image.storage is None: tag_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name)
return except model.DataModelException:
return None
model.storage.save_torrent_info(repo_image.storage, app.config['BITTORRENT_PIECE_SIZE'], return tag_image.docker_image_id
pieces_bytes)
@classmethod
def delete_tag(cls, namespace_name, repo_name, tag_name):
model.tag.delete_tag(namespace_name, repo_name, tag_name)
def image_ancestry(namespace_name, repo_name, image_id): @classmethod
""" Returns a list containing the full ancestry of Docker V1 IDs, in order, for the image with def load_token(cls, token):
the givne Docker V1 ID. try:
""" model.token.load_token_data(token)
try: return True
image = model.image.get_image_by_id(namespace_name, repo_name, image_id) except model.InvalidTokenException:
except model.InvalidImageException: return False
return None
parents = model.image.get_parent_images(namespace_name, repo_name, image) @classmethod
ancestry_docker_ids = [image.docker_image_id] def verify_robot(cls, username, token):
ancestry_docker_ids.extend([parent.docker_image_id for parent in parents]) try:
return ancestry_docker_ids return bool(model.user.verify_robot(username, token))
except model.InvalidRobotException:
return False
@classmethod
def change_user_password(cls, user, new_password):
model.user.change_password(user, new_password)
def repository_exists(namespace_name, repo_name): @classmethod
""" Returns whether the repository with the given name and namespace exists. """ def get_repository(cls, namespace_name, repo_name):
repo = model.repository.get_repository(namespace_name, repo_name) repo = model.repository.get_repository(namespace_name, repo_name)
return repo is not None if repo is None:
return None
return _repository_for_repo(repo)
@classmethod
def create_repository(cls, namespace_name, repo_name, user=None):
model.repository.create_repository(namespace_name, repo_name, user)
def create_or_link_image(username, namespace_name, repo_name, image_id, storage_location): @classmethod
""" Adds the given image to the given repository, by either linking to an existing image def repository_is_public(cls, namespace_name, repo_name):
visible to the user with the given username, or creating a new one if no existing image return model.repository.repository_is_public(namespace_name, repo_name)
matches.
"""
repo = model.repository.get_repository(namespace_name, repo_name)
model.image.find_create_or_link_image(image_id, repo, username, {}, storage_location)
@classmethod
def validate_oauth_token(cls, token):
return bool(model.oauth.validate_access_token(token))
def create_temp_hidden_tag(namespace_name, repo_name, image_id, expiration): @classmethod
""" Creates a hidden tag under the matching namespace pointing to the image with the given V1 def get_sorted_matching_repositories(cls, search_term, only_public, can_read, limit):
Docker ID. repos = model.repository.get_sorted_matching_repositories(search_term, only_public, can_read,
""" limit=limit)
repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id) return [_repository_for_repo(repo) for repo in repos]
if repo_image is None:
return
repo = repo_image.repository
model.tag.create_temporary_hidden_tag(repo, repo_image, expiration)
def list_tags(namespace_name, repo_name):
""" Returns all the tags defined in the repository with the given namespace and name. """
return model.tag.list_repository_tags(namespace_name, repo_name)
def create_or_update_tag(namespace_name, repo_name, image_id, tag_name):
""" Creates or updates a tag under the matching repository to point to the image with the given
Docker V1 ID.
"""
model.tag.create_or_update_tag(namespace_name, repo_name, tag_name, image_id)
def find_image_id_by_tag(namespace_name, repo_name, tag_name):
""" Returns the Docker V1 image ID for the HEAD image for the tag with the given name under
the matching repository, or None if none.
"""
try:
tag_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name)
except model.DataModelException:
return None
return tag_image.docker_image_id
def delete_tag(namespace_name, repo_name, tag_name):
""" Deletes the given tag from the given repository. """
model.tag.delete_tag(namespace_name, repo_name, tag_name)
def load_token(token):
""" Loads the data associated with the given (deprecated) access token, and, if found
returns True.
"""
try:
model.token.load_token_data(token)
return True
except model.InvalidTokenException:
return False
def verify_robot(username, token):
""" Returns True if the given robot username and token match an existing robot
account.
"""
try:
return bool(model.user.verify_robot(username, token))
except model.InvalidRobotException:
return False
def change_user_password(user, new_password):
""" Changes the password associated with the given user. """
model.user.change_password(user, new_password)
def get_repository(namespace_name, repo_name):
""" Returns the repository with the given name under the given namespace or None if none. """
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return None
return repository_for_repo(repo)
def create_repository(namespace_name, repo_name, user=None):
""" Creates a new repository under the given namespace with the given name, for the given user.
"""
model.repository.create_repository(namespace_name, repo_name, user)
def repository_is_public(namespace_name, repo_name):
""" Returns whether the repository with the given name under the given namespace is public.
If no matching repository was found, returns False.
"""
return model.repository.repository_is_public(namespace_name, repo_name)
def validate_oauth_token(token):
""" Returns whether the given OAuth token validates. """
return bool(model.oauth.validate_access_token(token))
def get_sorted_matching_repositories(search_term, only_public, can_read, limit):
""" Returns a sorted list of repositories matching the given search term. can_read is a callback
that will be invoked for each repository found, to filter results to only those visible to
the current user (if any).
"""
repos = model.repository.get_sorted_matching_repositories(search_term, only_public, can_read,
limit=limit)
return [repository_for_repo(repo) for repo in repos]

777
data/interfaces/v2.py
View file

@ -1,95 +1,74 @@
from collections import namedtuple
from namedlist import namedlist
from peewee import IntegrityError from peewee import IntegrityError
from data import model, database from data import model, database
from data.model import DataModelException from data.model import DataModelException
from image import Blob, BlobUpload, ManifestJSON, RepositoryReference, Tag
from image.docker.v1 import DockerV1Metadata from image.docker.v1 import DockerV1Metadata
from data.interfaces.common import repository_for_repo
_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws" _MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"
def create_repository(namespace_name, repo_name, creating_user=None):
""" Creates a new repository under the specified namespace with the given name. The user supplied class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
is the user creating the repository, if any.
""" """
return model.repository.create_repository(namespace_name, repo_name, creating_user) ManifestJSON represents a Manifest of any format.
def repository_is_public(namespace_name, repo_name):
""" Returns true if the repository with the given name under the given namespace has public
visibility.
""" """
return model.repository.repository_is_public(namespace_name, repo_name)
class Tag(namedtuple('Tag', ['name', 'repository'])):
def get_repository(namespace_name, repo_name):
""" Returns a repository tuple for the repository with the given name under the given namespace.
Returns None if no such repository was found.
""" """
repo = model.repository.get_repository(namespace_name, repo_name) Tag represents a user-facing alias for referencing a set of Manifests.
if repo is None:
return None
return repository_for_repo(repo)
def has_active_tag(namespace_name, repo_name, tag_name):
""" Returns whether there is an active tag for the tag with the given name under the matching
repository, if any, or None if none.
""" """
try:
model.tag.get_active_tag(namespace_name, repo_name, tag_name)
return True
except database.RepositoryTag.DoesNotExist:
return False
def get_manifest_by_tag(namespace_name, repo_name, tag_name): class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count',
""" Returns the current manifest for the tag with the given name under the matching repository, 'chunk_count', 'sha_state', 'location_name',
if any, or None if none. 'storage_metadata', 'piece_sha_state', 'piece_hashes',
'repo_namespace_name', 'repo_name'])):
""" """
try: BlobUpload represents the current state of an Blob being uploaded.
manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name)
return ManifestJSON(digest=manifest.digest, json=manifest.json_data, media_type=_MEDIA_TYPE)
except model.InvalidManifestException:
return None
def get_manifest_by_digest(namespace_name, repo_name, digest):
""" Returns the manifest matching the given digest under the matching repository, if any,
or None if none.
""" """
try:
manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest)
return ManifestJSON(digest=digest, json=manifest.json_data, media_type=_MEDIA_TYPE)
except model.InvalidManifestException:
return None
def delete_manifest_by_digest(namespace_name, repo_name, digest): class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])):
""" Deletes the manifest with the associated digest (if any) and returns all removed tags """
that pointed to that manifest. If the manifest was not found, returns an empty list. Blob represents an opaque binary blob saved to the storage system.
""" """
tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest)
def _tag_view(tag):
return Tag(
name=tag.name,
repository=RepositoryReference(
id=tag.repository_id,
name=repo_name,
namespace_name=namespace_name,
)
)
return [_tag_view(tag) for tag in tags] class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
"""
RepositoryReference represents a reference to a Repository, without its full metadata.
"""
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public'])):
"""
Repository represents a namespaced collection of tags.
"""
def _repository_for_repo(repo):
"""
Returns a Repository object representing the repo data model instance given.
"""
return Repository(
id=repo.id,
name=repo.name,
namespace_name=repo.namespace_user.username,
description=repo.description,
is_public=model.repository.is_repository_public(repo)
)
def _docker_v1_metadata(namespace_name, repo_name, repo_image): def _docker_v1_metadata(namespace_name, repo_name, repo_image):
""" Returns a DockerV1Metadata object for the given image under the repository with the given """
namespace and name. Note that the namespace and name are passed here as an optimization, Returns a DockerV1Metadata object for the given image under the repository with the given
and are *not checked* against the image. namespace and name. Note that the namespace and name are passed here as an optimization, and are
*not checked* against the image.
""" """
return DockerV1Metadata( return DockerV1Metadata(
namespace_name=namespace_name, namespace_name=namespace_name,
@ -98,264 +77,474 @@ def _docker_v1_metadata(namespace_name, repo_name, repo_image):
checksum=repo_image.v1_checksum, checksum=repo_image.v1_checksum,
content_checksum=repo_image.storage.content_checksum, content_checksum=repo_image.storage.content_checksum,
compat_json=repo_image.v1_json_metadata, compat_json=repo_image.v1_json_metadata,
created=repo_image.created, created=repo_image.created,
comment=repo_image.comment, comment=repo_image.comment,
command=repo_image.command, command=repo_image.command,
parent_image_id=None, # TODO: make sure this isn't needed anywhere, as it is expensive to lookup # TODO: make sure this isn't needed anywhere, as it is expensive to lookup
parent_image_id=None,
) )
def get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name): class DockerRegistryV2DataInterface(object):
""" Returns the Docker V1 metadata associated with the tag with the given name under the
matching repository, if any. If none, returns None.
""" """
try: Interface that represents all data store interactions required by a Docker Registry v1.
repo_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name, include_storage=True)
return _docker_v1_metadata(namespace_name, repo_name, repo_image)
except DataModelException:
return None
def get_docker_v1_metadata_by_image_id(namespace_name, repo_name, docker_image_ids):
""" Returns a map of Docker V1 metadata for each given image ID, matched under the repository
with the given namespace and name. Returns an empty map if the matching repository was not
found.
""" """
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return {}
images_query = model.image.lookup_repository_images(repo, docker_image_ids) @classmethod
return {image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image) def create_repository(cls, namespace_name, repo_name, creating_user=None):
for image in images_query} """
Creates a new repository under the specified namespace with the given name. The user supplied is
the user creating the repository, if any.
"""
raise NotImplementedError()
@classmethod
def repository_is_public(cls, namespace_name, repo_name):
"""
Returns true if the repository with the given name under the given namespace has public
visibility.
"""
raise NotImplementedError()
@classmethod
def get_repository(cls, namespace_name, repo_name):
"""
Returns a repository tuple for the repository with the given name under the given namespace.
Returns None if no such repository was found.
"""
raise NotImplementedError()
@classmethod
def has_active_tag(cls, namespace_name, repo_name, tag_name):
"""
Returns whether there is an active tag for the tag with the given name under the matching
repository, if any, or none if none.
"""
raise NotImplementedError()
@classmethod
def get_manifest_by_tag(cls, namespace_name, repo_name, tag_name):
"""
Returns the current manifest for the tag with the given name under the matching repository, if
any, or None if none.
"""
raise NotImplementedError()
@classmethod
def get_manifest_by_digest(cls, namespace_name, repo_name, digest):
"""
Returns the manifest matching the given digest under the matching repository, if any, or None if
none.
"""
raise NotImplementedError()
@classmethod
def delete_manifest_by_digest(cls, namespace_name, repo_name, digest):
"""
Deletes the manifest with the associated digest (if any) and returns all removed tags that
pointed to that manifest. If the manifest was not found, returns an empty list.
"""
raise NotImplementedError()
@classmethod
def get_docker_v1_metadata_by_tag(cls, namespace_name, repo_name, tag_name):
"""
Returns the Docker V1 metadata associated with the tag with the given name under the matching
repository, if any. If none, returns None.
"""
raise NotImplementedError()
@classmethod
def get_docker_v1_metadata_by_image_id(cls, namespace_name, repo_name, docker_image_ids):
"""
Returns a map of Docker V1 metadata for each given image ID, matched under the repository with
the given namespace and name. Returns an empty map if the matching repository was not found.
"""
raise NotImplementedError()
@classmethod
def get_parents_docker_v1_metadata(cls, namespace_name, repo_name, docker_image_id):
"""
Returns an ordered list containing the Docker V1 metadata for each parent of the image with the
given docker ID under the matching repository. Returns an empty list if the image was not found.
"""
raise NotImplementedError()
@classmethod
def create_manifest_and_update_tag(cls, namespace_name, repo_name, tag_name, manifest_digest,
manifest_bytes):
"""
Creates a new manifest with the given digest and byte data, and assigns the tag with the given
name under the matching repository to it.
"""
raise NotImplementedError()
@classmethod
def synthesize_v1_image(cls, repository, storage, image_id, created, comment, command,
compat_json, parent_image_id):
"""
Synthesizes a V1 image under the specified repository, pointing to the given storage and returns
the V1 metadata for the synthesized image.
"""
raise NotImplementedError()
@classmethod
def save_manifest(cls, namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest,
manifest_bytes):
"""
Saves a manifest pointing to the given leaf image, with the given manifest, under the matching
repository as a tag with the given name.
"""
raise NotImplementedError()
@classmethod
def repository_tags(cls, namespace_name, repo_name, limit, offset):
"""
Returns the active tags under the repository with the given name and namespace.
"""
raise NotImplementedError()
@classmethod
def get_visible_repositories(cls, username, limit, offset):
"""
Returns the repositories visible to the user with the given username, if any.
"""
raise NotImplementedError()
@classmethod
def create_blob_upload(cls, namespace_name, repo_name, upload_uuid, location_name,
storage_metadata):
"""
Creates a blob upload under the matching repository with the given UUID and metadata.
Returns whether the matching repository exists.
"""
raise NotImplementedError()
@classmethod
def blob_upload_by_uuid(cls, namespace_name, repo_name, upload_uuid):
"""
Searches for a blob upload with the given UUID under the given repository and returns it or None
if none.
"""
raise NotImplementedError()
@classmethod
def update_blob_upload(cls, blob_upload):
"""
Saves any changes to the blob upload object given to the backing data store.
Fields that can change:
- uncompressed_byte_count
- piece_hashes
- piece_sha_state
- storage_metadata
- byte_count
- chunk_count
- sha_state
"""
raise NotImplementedError()
@classmethod
def delete_blob_upload(cls, namespace_name, repo_name, uuid):
"""
Deletes the blob upload with the given uuid under the matching repository. If none, does
nothing.
"""
raise NotImplementedError()
@classmethod
def create_blob_and_temp_tag(cls, namespace_name, repo_name, blob_digest, blob_upload,
expiration_sec):
"""
Creates a blob and links a temporary tag with the specified expiration to it under the matching
repository.
"""
raise NotImplementedError()
@classmethod
def lookup_blobs_by_digest(cls, namespace_name, repo_name, digests):
"""
Returns all the blobs with matching digests found under the matching repository. If the
repository doesn't exist, returns {}.
"""
raise NotImplementedError()
@classmethod
def get_blob_by_digest(cls, namespace_name, repo_name, digest):
"""
Returns the blob with the given digest under the matching repository or None if none.
"""
raise NotImplementedError()
@classmethod
def save_bittorrent_pieces(cls, blob, piece_size, piece_bytes):
"""
Saves the BitTorrent piece hashes for the given blob.
"""
raise NotImplementedError()
@classmethod
def get_blob_path(cls, blob):
"""
Once everything is moved over, this could be in util.registry and not even touch the database.
"""
raise NotImplementedError()
def get_parents_docker_v1_metadata(namespace_name, repo_name, docker_image_id): class PreOCIModel(DockerRegistryV2DataInterface):
""" Returns an ordered list containing the Docker V1 metadata for each parent of the image
with the given docker ID under the matching repository. Returns an empty list if the image
was not found.
""" """
repo_image = model.image.get_repo_image(namespace_name, repo_name, docker_image_id) PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema
if repo_image is None: before it was changed to support the OCI specification.
return []
parents = model.image.get_parent_images(namespace_name, repo_name, repo_image)
return [_docker_v1_metadata(namespace_name, repo_name, image) for image in parents]
def create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest_digest,
manifest_bytes):
""" Creates a new manifest with the given digest and byte data, and assigns the tag with the
given name under the matching repository to it.
""" """
try: @classmethod
model.tag.associate_generated_tag_manifest(namespace_name, repo_name, tag_name, def create_repository(cls, namespace_name, repo_name, creating_user=None):
manifest_digest, manifest_bytes) return model.repository.create_repository(namespace_name, repo_name, creating_user)
except IntegrityError:
# It's already there!
pass
@classmethod
def repository_is_public(cls, namespace_name, repo_name):
return model.repository.repository_is_public(namespace_name, repo_name)
def synthesize_v1_image(repository, storage, image_id, created, comment, command, compat_json, @classmethod
parent_image_id): def get_repository(cls, namespace_name, repo_name):
""" Synthesizes a V1 image under the specified repository, pointing to the given storage repo = model.repository.get_repository(namespace_name, repo_name)
and returns the V1 metadata for the synthesized image. if repo is None:
""" return None
repo = model.repository.get_repository(repository.namespace_name, repository.name) return _repository_for_repo(repo)
if repo is None:
raise DataModelException('Unknown repository: %s/%s' % (repository.namespace_name,
repository.name))
parent_image = None @classmethod
if parent_image_id is not None: def has_active_tag(cls, namespace_name, repo_name, tag_name):
parent_image = model.image.get_image(repo, parent_image_id) try:
if parent_image is None: model.tag.get_active_tag(namespace_name, repo_name, tag_name)
raise DataModelException('Unknown parent image: %s' % parent_image_id) return True
except database.RepositoryTag.DoesNotExist:
return False
storage_obj = model.storage.get_storage_by_uuid(storage.uuid) @classmethod
if storage_obj is None: def get_manifest_by_tag(cls, namespace_name, repo_name, tag_name):
raise DataModelException('Unknown storage: %s' % storage.uuid) try:
manifest = model.tag.load_tag_manifest(namespace_name, repo_name, tag_name)
return ManifestJSON(digest=manifest.digest, json=manifest.json_data, media_type=_MEDIA_TYPE)
except model.InvalidManifestException:
return None
repo_image = model.image.synthesize_v1_image(repo, storage_obj, image_id, created, comment, @classmethod
command, compat_json, parent_image) def get_manifest_by_digest(cls, namespace_name, repo_name, digest):
return _docker_v1_metadata(repo.namespace_user.username, repo.name, repo_image) try:
manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, digest)
return ManifestJSON(digest=digest, json=manifest.json_data, media_type=_MEDIA_TYPE)
except model.InvalidManifestException:
return None
@classmethod
def save_manifest(namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest, def delete_manifest_by_digest(cls, namespace_name, repo_name, digest):
manifest_bytes): def _tag_view(tag):
""" Saves a manifest pointing to the given leaf image, with the given manifest, under the matching return Tag(
repository as a tag with the given name. name=tag.name,
""" repository=RepositoryReference(
model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_docker_id, id=tag.repository_id,
manifest_digest, manifest_bytes) name=repo_name,
namespace_name=namespace_name,
)
def repository_tags(namespace_name, repo_name, limit, offset):
""" Returns the active tags under the repository with the given name and namespace. """
tags_query = model.tag.list_repository_tags(namespace_name, repo_name)
tags_query = tags_query.limit(limit).offset(offset)
def _tag_view(tag):
return Tag(
name=tag.name,
repository=RepositoryReference(
id=tag.repository_id,
name=repo_name,
namespace_name=namespace_name,
) )
tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest)
return [_tag_view(tag) for tag in tags]
@classmethod
def get_docker_v1_metadata_by_tag(cls, namespace_name, repo_name, tag_name):
try:
repo_img = model.tag.get_tag_image(namespace_name, repo_name, tag_name, include_storage=True)
return _docker_v1_metadata(namespace_name, repo_name, repo_img)
except DataModelException:
return None
@classmethod
def get_docker_v1_metadata_by_image_id(cls, namespace_name, repo_name, docker_image_ids):
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return {}
images_query = model.image.lookup_repository_images(repo, docker_image_ids)
return {image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image)
for image in images_query}
@classmethod
def get_parents_docker_v1_metadata(cls, namespace_name, repo_name, docker_image_id):
repo_image = model.image.get_repo_image(namespace_name, repo_name, docker_image_id)
if repo_image is None:
return []
parents = model.image.get_parent_images(namespace_name, repo_name, repo_image)
return [_docker_v1_metadata(namespace_name, repo_name, image) for image in parents]
@classmethod
def create_manifest_and_update_tag(cls, namespace_name, repo_name, tag_name, manifest_digest,
manifest_bytes):
try:
model.tag.associate_generated_tag_manifest(namespace_name, repo_name, tag_name,
manifest_digest, manifest_bytes)
except IntegrityError:
# It's already there!
pass
@classmethod
def synthesize_v1_image(cls, repository, storage, image_id, created, comment, command,
compat_json, parent_image_id):
repo = model.repository.get_repository(repository.namespace_name, repository.name)
if repo is None:
raise DataModelException('Unknown repository: %s/%s' % (repository.namespace_name,
repository.name))
parent_image = None
if parent_image_id is not None:
parent_image = model.image.get_image(repo, parent_image_id)
if parent_image is None:
raise DataModelException('Unknown parent image: %s' % parent_image_id)
storage_obj = model.storage.get_storage_by_uuid(storage.uuid)
if storage_obj is None:
raise DataModelException('Unknown storage: %s' % storage.uuid)
repo_image = model.image.synthesize_v1_image(repo, storage_obj, image_id, created, comment,
command, compat_json, parent_image)
return _docker_v1_metadata(repo.namespace_user.username, repo.name, repo_image)
@classmethod
def save_manifest(cls, namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest,
manifest_bytes):
model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_docker_id,
manifest_digest, manifest_bytes)
@classmethod
def repository_tags(cls, namespace_name, repo_name, limit, offset):
def _tag_view(tag):
return Tag(
name=tag.name,
repository=RepositoryReference(
id=tag.repository_id,
name=repo_name,
namespace_name=namespace_name,
)
)
tags_query = model.tag.list_repository_tags(namespace_name, repo_name)
tags_query = tags_query.limit(limit).offset(offset)
return [_tag_view(tag) for tag in tags_query]
@classmethod
def get_visible_repositories(cls, username, limit, offset):
query = model.repository.get_visible_repositories(username, include_public=(username is None))
query = query.limit(limit).offset(offset)
return [_repository_for_repo(repo) for repo in query]
@classmethod
def create_blob_upload(cls, namespace_name, repo_name, upload_uuid, location_name,
storage_metadata):
try:
model.blob.initiate_upload(namespace_name, repo_name, upload_uuid, location_name,
storage_metadata)
return True
except database.Repository.DoesNotExist:
return False
@classmethod
def blob_upload_by_uuid(cls, namespace_name, repo_name, upload_uuid):
try:
found = model.blob.get_blob_upload(namespace_name, repo_name, upload_uuid)
except model.InvalidBlobUpload:
return None
return BlobUpload(
repo_namespace_name=namespace_name,
repo_name=repo_name,
uuid=upload_uuid,
byte_count=found.byte_count,
uncompressed_byte_count=found.uncompressed_byte_count,
chunk_count=found.chunk_count,
sha_state=found.sha_state,
piece_sha_state=found.piece_sha_state,
piece_hashes=found.piece_hashes,
location_name=found.location.name,
storage_metadata=found.storage_metadata,
) )
return [_tag_view(tag) for tag in tags_query] @classmethod
def update_blob_upload(cls, blob_upload):
# Lookup the blob upload object.
try:
blob_upload_record = model.blob.get_blob_upload(blob_upload.repo_namespace_name,
blob_upload.repo_name, blob_upload.uuid)
except model.InvalidBlobUpload:
return
blob_upload_record.uncompressed_byte_count = blob_upload.uncompressed_byte_count
blob_upload_record.piece_hashes = blob_upload.piece_hashes
blob_upload_record.piece_sha_state = blob_upload.piece_sha_state
blob_upload_record.storage_metadata = blob_upload.storage_metadata
blob_upload_record.byte_count = blob_upload.byte_count
blob_upload_record.chunk_count = blob_upload.chunk_count
blob_upload_record.sha_state = blob_upload.sha_state
blob_upload_record.save()
def get_visible_repositories(username, limit, offset): @classmethod
""" Returns the repositories visible to the user with the given username, if any. """ def delete_blob_upload(cls, namespace_name, repo_name, uuid):
query = model.repository.get_visible_repositories(username, include_public=(username is None)) try:
query = query.limit(limit).offset(offset) found = model.blob.get_blob_upload(namespace_name, repo_name, uuid)
return [repository_for_repo(repo) for repo in query] found.delete_instance()
except model.InvalidBlobUpload:
return
@classmethod
def create_blob_upload(namespace_name, repo_name, upload_uuid, location_name, storage_metadata): def create_blob_and_temp_tag(cls, namespace_name, repo_name, blob_digest, blob_upload,
""" Creates a blob upload under the matching repository with the given UUID and metadata. expiration_sec):
Returns whether the matching repository exists. location_obj = model.storage.get_image_location_for_name(blob_upload.location_name)
""" blob_record = model.blob.store_blob_record_and_temp_link(namespace_name, repo_name,
try: blob_digest,
model.blob.initiate_upload(namespace_name, repo_name, upload_uuid, location_name, location_obj.id,
storage_metadata) blob_upload.byte_count,
return True expiration_sec,
except database.Repository.DoesNotExist: blob_upload.uncompressed_byte_count)
return False
def blob_upload_by_uuid(namespace_name, repo_name, upload_uuid):
""" Searches for a blob upload with the given UUID under the given repository and returns it
or None if none.
"""
try:
found = model.blob.get_blob_upload(namespace_name, repo_name, upload_uuid)
except model.InvalidBlobUpload:
return None
return BlobUpload(
repo_namespace_name=namespace_name,
repo_name=repo_name,
uuid=upload_uuid,
byte_count=found.byte_count,
uncompressed_byte_count=found.uncompressed_byte_count,
chunk_count=found.chunk_count,
sha_state=found.sha_state,
piece_sha_state=found.piece_sha_state,
piece_hashes=found.piece_hashes,
location_name=found.location.name,
storage_metadata=found.storage_metadata,
)
def update_blob_upload(blob_upload):
""" Saves any changes to the blob upload object given to the backing data store.
Fields that can change:
- uncompressed_byte_count
- piece_hashes
- piece_sha_state
- storage_metadata
- byte_count
- chunk_count
- sha_state
"""
# Lookup the blob upload object.
try:
blob_upload_record = model.blob.get_blob_upload(blob_upload.repo_namespace_name,
blob_upload.repo_name, blob_upload.uuid)
except model.InvalidBlobUpload:
return
blob_upload_record.uncompressed_byte_count = blob_upload.uncompressed_byte_count
blob_upload_record.piece_hashes = blob_upload.piece_hashes
blob_upload_record.piece_sha_state = blob_upload.piece_sha_state
blob_upload_record.storage_metadata = blob_upload.storage_metadata
blob_upload_record.byte_count = blob_upload.byte_count
blob_upload_record.chunk_count = blob_upload.chunk_count
blob_upload_record.sha_state = blob_upload.sha_state
blob_upload_record.save()
def delete_blob_upload(namespace_name, repo_name, uuid):
""" Deletes the blob upload with the given uuid under the matching repository. If none, does
nothing.
"""
try:
found = model.blob.get_blob_upload(namespace_name, repo_name, uuid)
except model.InvalidBlobUpload:
return
found.delete_instance()
def create_blob_and_temp_tag(namespace_name, repo_name, blob_digest, blob_upload, expiration_sec):
""" Crates a blob and links a temporary tag with the specified expiration to it under the
matching repository.
"""
location_obj = model.storage.get_image_location_for_name(blob_upload.location_name)
blob_record = model.blob.store_blob_record_and_temp_link(namespace_name, repo_name,
blob_digest,
location_obj.id,
blob_upload.byte_count,
expiration_sec,
blob_upload.uncompressed_byte_count)
return Blob(
uuid=blob_record.uuid,
digest=blob_digest,
size=blob_upload.byte_count,
locations=[blob_upload.location_name],
)
def lookup_blobs_by_digest(namespace_name, repo_name, digests):
""" Returns all the blobs with matching digests found under the matching repository. If the
repository doesn't exist, returns {}.
"""
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return {}
def _blob_view(blob_record):
return Blob( return Blob(
uuid=blob_record.uuid, uuid=blob_record.uuid,
digest=blob_record.content_checksum, digest=blob_digest,
size=blob_record.image_size, size=blob_upload.byte_count,
locations=None, # Note: Locations is None in this case. locations=[blob_upload.location_name],
) )
query = model.storage.lookup_repo_storages_by_content_checksum(repo, digests) @classmethod
return {storage.content_checksum: _blob_view(storage) for storage in query} def lookup_blobs_by_digest(cls, namespace_name, repo_name, digests):
def _blob_view(blob_record):
return Blob(
uuid=blob_record.uuid,
digest=blob_record.content_checksum,
size=blob_record.image_size,
locations=None, # Note: Locations is None in this case.
)
repo = model.repository.get_repository(namespace_name, repo_name)
if repo is None:
return {}
query = model.storage.lookup_repo_storages_by_content_checksum(repo, digests)
return {storage.content_checksum: _blob_view(storage) for storage in query}
def get_blob_by_digest(namespace_name, repo_name, digest): @classmethod
""" Returns the blob with the given digest under the matching repository or None if none. """ def get_blob_by_digest(cls, namespace_name, repo_name, digest):
try: try:
blob_record = model.blob.get_repo_blob_by_digest(namespace_name, repo_name, digest) blob_record = model.blob.get_repo_blob_by_digest(namespace_name, repo_name, digest)
return Blob( return Blob(
uuid=blob_record.uuid, uuid=blob_record.uuid,
digest=digest, digest=digest,
size=blob_record.image_size, size=blob_record.image_size,
locations=blob_record.locations, locations=blob_record.locations,
) )
except model.BlobDoesNotExist: except model.BlobDoesNotExist:
return None return None
@classmethod
def save_bittorrent_pieces(cls, blob, piece_size, piece_bytes):
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
model.storage.save_torrent_info(blob_record, piece_size, piece_bytes)
def save_bittorrent_pieces(blob, piece_size, piece_bytes): @classmethod
""" Saves the BitTorrent piece hashes for the given blob. """ def get_blob_path(cls, blob):
blob_record = model.storage.get_storage_by_uuid(blob.uuid) blob_record = model.storage.get_storage_by_uuid(blob.uuid)
model.storage.save_torrent_info(blob_record, piece_size, piece_bytes) return model.storage.get_layer_path(blob_record)
def get_blob_path(blob):
# Once everything is moved over, this could be in util.registry and not even
# touch the database.
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
return model.storage.get_layer_path(blob_record)

23
endpoints/v1/index.py
View file

@ -6,9 +6,8 @@ from functools import wraps
from flask import request, make_response, jsonify, session from flask import request, make_response, jsonify, session
from data.interfaces import v1 from data.interfaces.v1 import PreOCIModel as model
from app import authentication, userevents, metric_queue from app import authentication, userevents, metric_queue
from app import authentication, userevents
from auth.auth import process_auth, generate_signed_token from auth.auth import process_auth, generate_signed_token
from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission, from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission,
@ -86,17 +85,17 @@ def create_user():
success = make_response('"Username or email already exists"', 400) success = make_response('"Username or email already exists"', 400)
if username == '$token': if username == '$token':
if v1.load_token(password): if model.load_token(password):
return success return success
abort(400, 'Invalid access token.', issue='invalid-access-token') abort(400, 'Invalid access token.', issue='invalid-access-token')
elif username == '$oauthtoken': elif username == '$oauthtoken':
if v1.validate_oauth_token(password): if model.validate_oauth_token(password):
return success return success
abort(400, 'Invalid oauth access token.', issue='invalid-oauth-access-token') abort(400, 'Invalid oauth access token.', issue='invalid-oauth-access-token')
elif '+' in username: elif '+' in username:
if v1.verify_robot(username, password): if model.verify_robot(username, password):
return success return success
abort(400, 'Invalid robot account or password.', issue='robot-login-failure') abort(400, 'Invalid robot account or password.', issue='robot-login-failure')
@ -147,7 +146,7 @@ def update_user(username):
if 'password' in update_request: if 'password' in update_request:
logger.debug('Updating user password') logger.debug('Updating user password')
v1.change_user_password(get_authenticated_user(), update_request['password']) model.change_user_password(get_authenticated_user(), update_request['password'])
return jsonify({ return jsonify({
'username': get_authenticated_user().username, 'username': get_authenticated_user().username,
@ -167,7 +166,7 @@ def create_repository(namespace_name, repo_name):
abort(400, message='Invalid repository name. Repository names cannot contain slashes.') abort(400, message='Invalid repository name. Repository names cannot contain slashes.')
logger.debug('Looking up repository %s/%s', namespace_name, repo_name) logger.debug('Looking up repository %s/%s', namespace_name, repo_name)
repo = v1.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
logger.debug('Found repository %s/%s', namespace_name, repo_name) logger.debug('Found repository %s/%s', namespace_name, repo_name)
if not repo and get_authenticated_user() is None: if not repo and get_authenticated_user() is None:
@ -195,7 +194,7 @@ def create_repository(namespace_name, repo_name):
logger.debug('Creating repository %s/%s with owner: %s', namespace_name, repo_name, logger.debug('Creating repository %s/%s with owner: %s', namespace_name, repo_name,
get_authenticated_user().username) get_authenticated_user().username)
v1.create_repository(namespace_name, repo_name, get_authenticated_user()) model.create_repository(namespace_name, repo_name, get_authenticated_user())
if get_authenticated_user(): if get_authenticated_user():
user_event_data = { user_event_data = {
@ -220,7 +219,7 @@ def update_images(namespace_name, repo_name):
if permission.can(): if permission.can():
logger.debug('Looking up repository') logger.debug('Looking up repository')
repo = v1.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if not repo: if not repo:
# Make sure the repo actually exists. # Make sure the repo actually exists.
abort(404, message='Unknown repository', issue='unknown-repo') abort(404, message='Unknown repository', issue='unknown-repo')
@ -250,10 +249,10 @@ def get_repository_images(namespace_name, repo_name):
permission = ReadRepositoryPermission(namespace_name, repo_name) permission = ReadRepositoryPermission(namespace_name, repo_name)
# TODO invalidate token? # TODO invalidate token?
if permission.can() or v1.repository_is_public(namespace_name, repo_name): if permission.can() or model.repository_is_public(namespace_name, repo_name):
# We can't rely on permissions to tell us if a repo exists anymore # We can't rely on permissions to tell us if a repo exists anymore
logger.debug('Looking up repository') logger.debug('Looking up repository')
repo = v1.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if not repo: if not repo:
abort(404, message='Unknown repository', issue='unknown-repo') abort(404, message='Unknown repository', issue='unknown-repo')
@ -319,7 +318,7 @@ def _conduct_repo_search(username, query, results):
return ReadRepositoryPermission(repo.namespace_name, repo.name).can() return ReadRepositoryPermission(repo.namespace_name, repo.name).can()
only_public = username is None only_public = username is None
matching_repos = v1.get_sorted_matching_repositories(query, only_public, can_read, limit=5) matching_repos = model.get_sorted_matching_repositories(query, only_public, can_read, limit=5)
for repo in matching_repos: for repo in matching_repos:
results.append({ results.append({

70
endpoints/v1/registry.py
View file

@ -14,7 +14,7 @@ from auth.permissions import (ReadRepositoryPermission,
ModifyRepositoryPermission) ModifyRepositoryPermission)
from auth.registry_jwt_auth import get_granted_username from auth.registry_jwt_auth import get_granted_username
from data import model, database from data import model, database
from data.interfaces import v1 from data.interfaces.v1 import PreOCIModel as model
from digest import checksums from digest import checksums
from endpoints.v1 import v1_bp from endpoints.v1 import v1_bp
from endpoints.decorators import anon_protect from endpoints.decorators import anon_protect
@ -30,7 +30,7 @@ logger = logging.getLogger(__name__)
def _finish_image(namespace, repository, image_id): def _finish_image(namespace, repository, image_id):
# Checksum is ok, we remove the marker # Checksum is ok, we remove the marker
blob_ref = v1.update_image_uploading(namespace, repository, image_id, False) blob_ref = model.update_image_uploading(namespace, repository, image_id, False)
# Send a job to the work queue to replicate the image layer. # Send a job to the work queue to replicate the image layer.
queue_storage_replication(namespace, blob_ref) queue_storage_replication(namespace, blob_ref)
@ -41,7 +41,7 @@ def require_completion(f):
@wraps(f) @wraps(f)
def wrapper(namespace, repository, *args, **kwargs): def wrapper(namespace, repository, *args, **kwargs):
image_id = kwargs['image_id'] image_id = kwargs['image_id']
if v1.is_image_uploading(namespace, repository, image_id): if model.is_image_uploading(namespace, repository, image_id):
abort(400, 'Image %(image_id)s is being uploaded, retry later', abort(400, 'Image %(image_id)s is being uploaded, retry later',
issue='upload-in-progress', image_id=image_id) issue='upload-in-progress', image_id=image_id)
return f(namespace, repository, *args, **kwargs) return f(namespace, repository, *args, **kwargs)
@ -82,9 +82,9 @@ def head_image_layer(namespace, repository, image_id, headers):
permission = ReadRepositoryPermission(namespace, repository) permission = ReadRepositoryPermission(namespace, repository)
logger.debug('Checking repo permissions') logger.debug('Checking repo permissions')
if permission.can() or model.repository.repository_is_public(namespace, repository): if permission.can() or model.repository_is_public(namespace, repository):
logger.debug('Looking up placement locations') logger.debug('Looking up placement locations')
locations = v1.placement_locations_docker_v1(namespace, repository, image_id) locations = model.placement_locations_docker_v1(namespace, repository, image_id)
if locations is None: if locations is None:
logger.debug('Could not find any blob placement locations') logger.debug('Could not find any blob placement locations')
abort(404, 'Image %(image_id)s not found', issue='unknown-image', abort(404, 'Image %(image_id)s not found', issue='unknown-image',
@ -115,11 +115,9 @@ def get_image_layer(namespace, repository, image_id, headers):
permission = ReadRepositoryPermission(namespace, repository) permission = ReadRepositoryPermission(namespace, repository)
logger.debug('Checking repo permissions') logger.debug('Checking repo permissions')
if permission.can() or model.repository.repository_is_public(namespace, repository): if permission.can() or model.repository_is_public(namespace, repository):
logger.debug('Looking up placement locations and path') logger.debug('Looking up placement locations and path')
locations, path = v1.placement_locations_and_path_docker_v1(namespace, locations, path = model.placement_locations_and_path_docker_v1(namespace, repository, image_id)
repository,
image_id)
if not locations or not path: if not locations or not path:
abort(404, 'Image %(image_id)s not found', issue='unknown-image', abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id) image_id=image_id)
@ -154,7 +152,7 @@ def put_image_layer(namespace, repository, image_id):
abort(403) abort(403)
logger.debug('Retrieving image') logger.debug('Retrieving image')
if v1.storage_exists(namespace, repository, image_id): if model.storage_exists(namespace, repository, image_id):
exact_abort(409, 'Image already exists') exact_abort(409, 'Image already exists')
logger.debug('Storing layer data') logger.debug('Storing layer data')
@ -184,7 +182,7 @@ def put_image_layer(namespace, repository, image_id):
sr.add_handler(piece_hasher.update) sr.add_handler(piece_hasher.update)
# Add a handler which computes the checksum. # Add a handler which computes the checksum.
v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) v1_metadata = model.docker_v1_metadata(namespace, repository, image_id)
h, sum_hndlr = checksums.simple_checksum_handler(v1_metadata.compat_json) h, sum_hndlr = checksums.simple_checksum_handler(v1_metadata.compat_json)
sr.add_handler(sum_hndlr) sr.add_handler(sum_hndlr)
@ -193,7 +191,7 @@ def put_image_layer(namespace, repository, image_id):
sr.add_handler(content_sum_hndlr) sr.add_handler(content_sum_hndlr)
# Stream write the data to storage. # Stream write the data to storage.
locations, path = v1.placement_locations_and_path_docker_v1(namespace, repository, image_id) locations, path = model.placement_locations_and_path_docker_v1(namespace, repository, image_id)
with database.CloseForLongOperation(app.config): with database.CloseForLongOperation(app.config):
try: try:
store.stream_write(locations, path, sr) store.stream_write(locations, path, sr)
@ -202,11 +200,11 @@ def put_image_layer(namespace, repository, image_id):
abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id) abort(520, 'Image %(image_id)s could not be written. Please try again.', image_id=image_id)
# Save the size of the image. # Save the size of the image.
v1.update_image_sizes(namespace, repository, image_id, size_info.compressed_size, model.update_image_sizes(namespace, repository, image_id, size_info.compressed_size,
size_info.uncompressed_size) size_info.uncompressed_size)
# Save the BitTorrent pieces. # Save the BitTorrent pieces.
v1.create_bittorrent_pieces(namespace, repository, image_id, piece_hasher.final_piece_hashes()) model.create_bittorrent_pieces(namespace, repository, image_id, piece_hasher.final_piece_hashes())
# Append the computed checksum. # Append the computed checksum.
csums = [] csums = []
@ -271,7 +269,7 @@ def put_image_checksum(namespace, repository, image_id):
issue='missing-checksum-cookie', image_id=image_id) issue='missing-checksum-cookie', image_id=image_id)
logger.debug('Looking up repo image') logger.debug('Looking up repo image')
v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) v1_metadata = model.docker_v1_metadata(namespace, repository, image_id)
if not v1_metadata: if not v1_metadata:
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
@ -280,7 +278,7 @@ def put_image_checksum(namespace, repository, image_id):
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id) abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
logger.debug('Marking image path') logger.debug('Marking image path')
if not v1.is_image_uploading(namespace, repository, image_id): if not model.is_image_uploading(namespace, repository, image_id):
abort(409, 'Cannot set checksum for image %(image_id)s', abort(409, 'Cannot set checksum for image %(image_id)s',
issue='image-write-error', image_id=image_id) issue='image-write-error', image_id=image_id)
@ -291,7 +289,7 @@ def put_image_checksum(namespace, repository, image_id):
if len(checksum_parts) != 2: if len(checksum_parts) != 2:
abort(400, 'Invalid checksum format') abort(400, 'Invalid checksum format')
v1.store_docker_v1_checksums(namespace, repository, image_id, checksum, content_checksum) model.store_docker_v1_checksums(namespace, repository, image_id, checksum, content_checksum)
if checksum not in session.get('checksum', []): if checksum not in session.get('checksum', []):
logger.debug('session checksums: %s', session.get('checksum', [])) logger.debug('session checksums: %s', session.get('checksum', []))
@ -315,16 +313,16 @@ def put_image_checksum(namespace, repository, image_id):
def get_image_json(namespace, repository, image_id, headers): def get_image_json(namespace, repository, image_id, headers):
logger.debug('Checking repo permissions') logger.debug('Checking repo permissions')
permission = ReadRepositoryPermission(namespace, repository) permission = ReadRepositoryPermission(namespace, repository)
if not permission.can() and not model.repository.repository_is_public(namespace, repository): if not permission.can() and not model.repository_is_public(namespace, repository):
abort(403) abort(403)
logger.debug('Looking up repo image') logger.debug('Looking up repo image')
v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) v1_metadata = model.docker_v1_metadata(namespace, repository, image_id)
if v1_metadata is None: if v1_metadata is None:
flask_abort(404) flask_abort(404)
logger.debug('Looking up repo layer size') logger.debug('Looking up repo layer size')
size = v1.get_image_size(namespace, repository, image_id) size = model.get_image_size(namespace, repository, image_id)
if size is not None: if size is not None:
# Note: X-Docker-Size is optional and we *can* end up with a NULL image_size, # Note: X-Docker-Size is optional and we *can* end up with a NULL image_size,
# so handle this case rather than failing. # so handle this case rather than failing.
@ -344,10 +342,10 @@ def get_image_json(namespace, repository, image_id, headers):
def get_image_ancestry(namespace, repository, image_id, headers): def get_image_ancestry(namespace, repository, image_id, headers):
logger.debug('Checking repo permissions') logger.debug('Checking repo permissions')
permission = ReadRepositoryPermission(namespace, repository) permission = ReadRepositoryPermission(namespace, repository)
if not permission.can() and not model.repository.repository_is_public(namespace, repository): if not permission.can() and not model.repository_is_public(namespace, repository):
abort(403) abort(403)
ancestry_docker_ids = v1.image_ancestry(namespace, repository, image_id) ancestry_docker_ids = model.image_ancestry(namespace, repository, image_id)
if ancestry_docker_ids is None: if ancestry_docker_ids is None:
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id) abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
@ -388,37 +386,39 @@ def put_image_json(namespace, repository, image_id):
logger.debug('Looking up repo image') logger.debug('Looking up repo image')
if not v1.repository_exists(namespace, repository): if not model.repository_exists(namespace, repository):
abort(404, 'Repository does not exist: %(namespace)s/%(repository)s', issue='no-repo', abort(404, 'Repository does not exist: %(namespace)s/%(repository)s', issue='no-repo',
namespace=namespace, repository=repository) namespace=namespace, repository=repository)
v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) v1_metadata = model.docker_v1_metadata(namespace, repository, image_id)
if v1_metadata is None: if v1_metadata is None:
username = get_authenticated_user() and get_authenticated_user().username username = get_authenticated_user() and get_authenticated_user().username
if not username: if not username:
username = get_granted_username() username = get_granted_username()
logger.debug('Image not found, creating or linking image with initiating user context: %s', username) logger.debug('Image not found, creating or linking image with initiating user context: %s',
v1.create_or_link_image(username, namespace, repository, image_id, store.preferred_locations[0]) username)
v1_metadata = v1.docker_v1_metadata(namespace, repository, image_id) location_pref = store.preferred_locations[0]
model.create_or_link_image(username, namespace, repository, image_id, location_pref)
v1_metadata = model.docker_v1_metadata(namespace, repository, image_id)
# Create a temporary tag to prevent this image from getting garbage collected while the push # Create a temporary tag to prevent this image from getting garbage collected while the push
# is in progress. # is in progress.
v1.create_temp_hidden_tag(namespace, repository, image_id, model.create_temp_hidden_tag(namespace, repository, image_id,
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']) app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
parent_id = data.get('parent', None) parent_id = data.get('parent', None)
if parent_id: if parent_id:
logger.debug('Looking up parent image') logger.debug('Looking up parent image')
if v1.docker_v1_metadata(namespace, repository, parent_id) is None: if model.docker_v1_metadata(namespace, repository, parent_id) is None:
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s', abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
issue='invalid-request', image_id=image_id, parent_id=parent_id) issue='invalid-request', image_id=image_id, parent_id=parent_id)
logger.debug('Checking if image already exists') logger.debug('Checking if image already exists')
if v1_metadata and not v1.is_image_uploading(namespace, repository, image_id): if v1_metadata and not model.is_image_uploading(namespace, repository, image_id):
exact_abort(409, 'Image already exists') exact_abort(409, 'Image already exists')
v1.update_image_uploading(namespace, repository, image_id, True) model.update_image_uploading(namespace, repository, image_id, True)
# If we reach that point, it means that this is a new image or a retry # If we reach that point, it means that this is a new image or a retry
# on a failed push, save the metadata # on a failed push, save the metadata
@ -426,7 +426,7 @@ def put_image_json(namespace, repository, image_id):
command = json.dumps(command_list) if command_list else None command = json.dumps(command_list) if command_list else None
logger.debug('Setting image metadata') logger.debug('Setting image metadata')
v1.update_docker_v1_metadata(namespace, repository, image_id, data.get('created'), model.update_docker_v1_metadata(namespace, repository, image_id, data.get('created'),
data.get('comment'), command, uploaded_metadata, parent_id) data.get('comment'), command, uploaded_metadata, parent_id)
return make_response('true', 200) return make_response('true', 200)

17
endpoints/v1/tag.py
View file

@ -9,7 +9,7 @@ from auth.auth import process_auth
from auth.permissions import (ReadRepositoryPermission, from auth.permissions import (ReadRepositoryPermission,
ModifyRepositoryPermission) ModifyRepositoryPermission)
from data import model from data import model
from data.interfaces import v1 from data.interfaces.v1 import PreOCIModel as model
from endpoints.common import parse_repository_name from endpoints.common import parse_repository_name
from endpoints.decorators import anon_protect from endpoints.decorators import anon_protect
from endpoints.v1 import v1_bp from endpoints.v1 import v1_bp
@ -26,8 +26,8 @@ logger = logging.getLogger(__name__)
def get_tags(namespace_name, repo_name): def get_tags(namespace_name, repo_name):
permission = ReadRepositoryPermission(namespace_name, repo_name) permission = ReadRepositoryPermission(namespace_name, repo_name)
if permission.can() or model.repository.repository_is_public(namespace_name, repo_name): if permission.can() or model.repository_is_public(namespace_name, repo_name):
tags = v1.list_tags(namespace_name, repo_name) tags = model.list_tags(namespace_name, repo_name)
tag_map = {tag.name: tag.image.docker_image_id for tag in tags} tag_map = {tag.name: tag.image.docker_image_id for tag in tags}
return jsonify(tag_map) return jsonify(tag_map)
@ -41,8 +41,8 @@ def get_tags(namespace_name, repo_name):
def get_tag(namespace_name, repo_name, tag): def get_tag(namespace_name, repo_name, tag):
permission = ReadRepositoryPermission(namespace_name, repo_name) permission = ReadRepositoryPermission(namespace_name, repo_name)
if permission.can() or model.repository.repository_is_public(namespace_name, repo_name): if permission.can() or model.repository_is_public(namespace_name, repo_name):
image_id = v1.find_image_id_by_tag(namespace_name, repo_name, tag) image_id = model.find_image_id_by_tag(namespace_name, repo_name, tag)
if image_id is None: if image_id is None:
abort(404) abort(404)
@ -65,7 +65,7 @@ def put_tag(namespace_name, repo_name, tag):
abort(400, TAG_ERROR) abort(400, TAG_ERROR)
image_id = json.loads(request.data) image_id = json.loads(request.data)
v1.create_or_update_tag(namespace_name, repo_name, image_id, tag) model.create_or_update_tag(namespace_name, repo_name, image_id, tag)
# Store the updated tag. # Store the updated tag.
if 'pushed_tags' not in session: if 'pushed_tags' not in session:
@ -86,9 +86,8 @@ def delete_tag(namespace_name, repo_name, tag):
permission = ModifyRepositoryPermission(namespace_name, repo_name) permission = ModifyRepositoryPermission(namespace_name, repo_name)
if permission.can(): if permission.can():
v1.delete_tag(namespace_name, repo_name, tag) model.delete_tag(namespace_name, repo_name, tag)
track_and_log('delete_tag', model.repository.get_repository(namespace_name, repo_name), track_and_log('delete_tag', model.get_repository(namespace_name, repo_name), tag=tag)
tag=tag)
return make_response('Deleted', 200) return make_response('Deleted', 200)
abort(403) abort(403)

34
endpoints/v2/blob.py
View file

@ -8,7 +8,7 @@ import resumablehashlib
from app import storage, app from app import storage, app
from auth.registry_jwt_auth import process_registry_jwt_auth from auth.registry_jwt_auth import process_registry_jwt_auth
from data import database from data import database
from data.interfaces import v2 from data.interfaces.v2 import PreOCIModel as model
from digest import digest_tools from digest import digest_tools
from endpoints.common import parse_repository_name from endpoints.common import parse_repository_name
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream
@ -42,7 +42,7 @@ class _InvalidRangeHeader(Exception):
@cache_control(max_age=31436000) @cache_control(max_age=31436000)
def check_blob_exists(namespace_name, repo_name, digest): def check_blob_exists(namespace_name, repo_name, digest):
# Find the blob. # Find the blob.
blob = v2.get_blob_by_digest(namespace_name, repo_name, digest) blob = model.get_blob_by_digest(namespace_name, repo_name, digest)
if blob is None: if blob is None:
raise BlobUnknown() raise BlobUnknown()
@ -69,7 +69,7 @@ def check_blob_exists(namespace_name, repo_name, digest):
@cache_control(max_age=31536000) @cache_control(max_age=31536000)
def download_blob(namespace_name, repo_name, digest): def download_blob(namespace_name, repo_name, digest):
# Find the blob. # Find the blob.
blob = v2.get_blob_by_digest(namespace_name, repo_name, digest) blob = model.get_blob_by_digest(namespace_name, repo_name, digest)
if blob is None: if blob is None:
raise BlobUnknown() raise BlobUnknown()
@ -81,7 +81,7 @@ def download_blob(namespace_name, repo_name, digest):
headers['Accept-Ranges'] = 'bytes' headers['Accept-Ranges'] = 'bytes'
# Find the storage path for the blob. # Find the storage path for the blob.
path = v2.get_blob_path(blob) path = model.get_blob_path(blob)
# Short-circuit by redirecting if the storage supports it. # Short-circuit by redirecting if the storage supports it.
logger.debug('Looking up the direct download URL for path: %s', path) logger.debug('Looking up the direct download URL for path: %s', path)
@ -115,8 +115,8 @@ def start_blob_upload(namespace_name, repo_name):
# Begin the blob upload process in the database and storage. # Begin the blob upload process in the database and storage.
location_name = storage.preferred_locations[0] location_name = storage.preferred_locations[0]
new_upload_uuid, upload_metadata = storage.initiate_chunked_upload(location_name) new_upload_uuid, upload_metadata = storage.initiate_chunked_upload(location_name)
repository_exists = v2.create_blob_upload(namespace_name, repo_name, new_upload_uuid, repository_exists = model.create_blob_upload(namespace_name, repo_name, new_upload_uuid,
location_name, upload_metadata) location_name, upload_metadata)
if not repository_exists: if not repository_exists:
raise NameUnknown() raise NameUnknown()
@ -135,7 +135,7 @@ def start_blob_upload(namespace_name, repo_name):
# The user plans to send us the entire body right now. # The user plans to send us the entire body right now.
# Find the upload. # Find the upload.
blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, new_upload_uuid) blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, new_upload_uuid)
if blob_upload is None: if blob_upload is None:
raise BlobUploadUnknown() raise BlobUploadUnknown()
@ -146,7 +146,7 @@ def start_blob_upload(namespace_name, repo_name):
_abort_range_not_satisfiable(blob_upload.byte_count, new_upload_uuid) _abort_range_not_satisfiable(blob_upload.byte_count, new_upload_uuid)
# Save the upload state to the database. # Save the upload state to the database.
v2.update_blob_upload(updated_blob_upload) model.update_blob_upload(updated_blob_upload)
# Finalize the upload process in the database and storage. # Finalize the upload process in the database and storage.
_finish_upload(namespace_name, repo_name, updated_blob_upload, digest) _finish_upload(namespace_name, repo_name, updated_blob_upload, digest)
@ -168,7 +168,7 @@ def start_blob_upload(namespace_name, repo_name):
@require_repo_write @require_repo_write
@anon_protect @anon_protect
def fetch_existing_upload(namespace_name, repo_name, upload_uuid): def fetch_existing_upload(namespace_name, repo_name, upload_uuid):
blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
if blob_upload is None: if blob_upload is None:
raise BlobUploadUnknown() raise BlobUploadUnknown()
@ -188,7 +188,7 @@ def fetch_existing_upload(namespace_name, repo_name, upload_uuid):
@anon_protect @anon_protect
def upload_chunk(namespace_name, repo_name, upload_uuid): def upload_chunk(namespace_name, repo_name, upload_uuid):
# Find the upload. # Find the upload.
blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
if blob_upload is None: if blob_upload is None:
raise BlobUploadUnknown() raise BlobUploadUnknown()
@ -199,7 +199,7 @@ def upload_chunk(namespace_name, repo_name, upload_uuid):
_abort_range_not_satisfiable(blob_upload.byte_count, upload_uuid) _abort_range_not_satisfiable(blob_upload.byte_count, upload_uuid)
# Save the upload state to the database. # Save the upload state to the database.
v2.update_blob_upload(updated_blob_upload) model.update_blob_upload(updated_blob_upload)
# Write the response to the client. # Write the response to the client.
return Response( return Response(
@ -224,7 +224,7 @@ def monolithic_upload_or_last_chunk(namespace_name, repo_name, upload_uuid):
raise BlobUploadInvalid(detail={'reason': 'Missing digest arg on monolithic upload'}) raise BlobUploadInvalid(detail={'reason': 'Missing digest arg on monolithic upload'})
# Find the upload. # Find the upload.
blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
if blob_upload is None: if blob_upload is None:
raise BlobUploadUnknown() raise BlobUploadUnknown()
@ -254,13 +254,13 @@ def monolithic_upload_or_last_chunk(namespace_name, repo_name, upload_uuid):
@require_repo_write @require_repo_write
@anon_protect @anon_protect
def cancel_upload(namespace_name, repo_name, upload_uuid): def cancel_upload(namespace_name, repo_name, upload_uuid):
blob_upload = v2.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid) blob_upload = model.blob_upload_by_uuid(namespace_name, repo_name, upload_uuid)
if blob_upload is None: if blob_upload is None:
raise BlobUploadUnknown() raise BlobUploadUnknown()
# We delete the record for the upload first, since if the partial upload in # We delete the record for the upload first, since if the partial upload in
# storage fails to delete, it doesn't break anything. # storage fails to delete, it doesn't break anything.
v2.delete_blob_upload(namespace_name, repo_name, upload_uuid) model.delete_blob_upload(namespace_name, repo_name, upload_uuid)
storage.cancel_chunked_upload({blob_upload.location_name}, blob_upload.uuid, storage.cancel_chunked_upload({blob_upload.location_name}, blob_upload.uuid,
blob_upload.storage_metadata) blob_upload.storage_metadata)
@ -471,7 +471,7 @@ def _finalize_blob_database(namespace_name, repo_name, blob_upload, digest, alre
database's perspective. database's perspective.
""" """
# Create the blob and temporarily tag it. # Create the blob and temporarily tag it.
blob_storage = v2.create_blob_and_temp_tag( blob_storage = model.create_blob_and_temp_tag(
namespace_name, namespace_name,
repo_name, repo_name,
digest, digest,
@ -482,10 +482,10 @@ def _finalize_blob_database(namespace_name, repo_name, blob_upload, digest, alre
# If it doesn't already exist, create the BitTorrent pieces for the blob. # If it doesn't already exist, create the BitTorrent pieces for the blob.
if blob_upload.piece_sha_state is not None and not already_existed: if blob_upload.piece_sha_state is not None and not already_existed:
piece_bytes = blob_upload.piece_hashes + blob_upload.piece_sha_state.digest() piece_bytes = blob_upload.piece_hashes + blob_upload.piece_sha_state.digest()
v2.save_bittorrent_pieces(blob_storage, app.config['BITTORRENT_PIECE_SIZE'], piece_bytes) model.save_bittorrent_pieces(blob_storage, app.config['BITTORRENT_PIECE_SIZE'], piece_bytes)
# Delete the blob upload. # Delete the blob upload.
v2.delete_blob_upload(namespace_name, repo_name, blob_upload.uuid) model.delete_blob_upload(namespace_name, repo_name, blob_upload.uuid)
def _finish_upload(namespace_name, repo_name, blob_upload, digest): def _finish_upload(namespace_name, repo_name, blob_upload, digest):

4
endpoints/v2/catalog.py
View file

@ -3,7 +3,7 @@ from flask import jsonify
from auth.registry_jwt_auth import process_registry_jwt_auth, get_granted_entity from auth.registry_jwt_auth import process_registry_jwt_auth, get_granted_entity
from endpoints.decorators import anon_protect from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, paginate from endpoints.v2 import v2_bp, paginate
from data.interfaces import v2 from data.interfaces.v2 import PreOCIModel as model
@v2_bp.route('/_catalog', methods=['GET']) @v2_bp.route('/_catalog', methods=['GET'])
@process_registry_jwt_auth() @process_registry_jwt_auth()
@ -15,7 +15,7 @@ def catalog_search(limit, offset, pagination_callback):
if entity: if entity:
username = entity.user.username username = entity.user.username
visible_repositories = v2.get_visible_repositories(username, limit+1, offset) visible_repositories = model.get_visible_repositories(username, limit+1, offset)
response = jsonify({ response = jsonify({
'repositories': ['%s/%s' % (repo.namespace_name, repo.name) 'repositories': ['%s/%s' % (repo.namespace_name, repo.name)
for repo in visible_repositories][0:limit], for repo in visible_repositories][0:limit],

38
endpoints/v2/manifest.py
View file

@ -8,8 +8,7 @@ import features
from app import docker_v2_signing_key, app, metric_queue from app import docker_v2_signing_key, app, metric_queue
from auth.registry_jwt_auth import process_registry_jwt_auth from auth.registry_jwt_auth import process_registry_jwt_auth
from data import model from data.interfaces.v2 import PreOCIModel as model
from data.interfaces import v2
from digest import digest_tools from digest import digest_tools
from endpoints.common import parse_repository_name from endpoints.common import parse_repository_name
from endpoints.decorators import anon_protect from endpoints.decorators import anon_protect
@ -24,6 +23,7 @@ from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES
from util.registry.replication import queue_storage_replication from util.registry.replication import queue_storage_replication
from util.names import VALID_TAG_PATTERN from util.names import VALID_TAG_PATTERN
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -37,9 +37,9 @@ MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)
@require_repo_read @require_repo_read
@anon_protect @anon_protect
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref): def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
manifest = v2.get_manifest_by_tag(namespace_name, repo_name, manifest_ref) manifest = model.get_manifest_by_tag(namespace_name, repo_name, manifest_ref)
if manifest is None: if manifest is None:
has_tag = v2.has_active_tag(namespace_name, repo_name, manifest_ref) has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref)
if not has_tag: if not has_tag:
raise ManifestUnknown() raise ManifestUnknown()
@ -47,7 +47,7 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
if manifest is None: if manifest is None:
raise ManifestUnknown() raise ManifestUnknown()
repo = v2.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if repo is not None: if repo is not None:
track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01) track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01)
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2']) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
@ -65,12 +65,12 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
@require_repo_read @require_repo_read
@anon_protect @anon_protect
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref): def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
manifest = v2.get_manifest_by_digest(namespace_name, repo_name, manifest_ref) manifest = model.get_manifest_by_digest(namespace_name, repo_name, manifest_ref)
if manifest is None: if manifest is None:
# Without a tag name to reference, we can't make an attempt to generate the manifest # Without a tag name to reference, we can't make an attempt to generate the manifest
raise ManifestUnknown() raise ManifestUnknown()
repo = v2.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if repo is not None: if repo is not None:
track_and_log('pull_repo', repo) track_and_log('pull_repo', repo)
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2']) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
@ -137,7 +137,7 @@ def _write_manifest(namespace_name, repo_name, manifest):
raise NameInvalid() raise NameInvalid()
# Ensure that the repository exists. # Ensure that the repository exists.
repo = v2.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if repo is None: if repo is None:
raise NameInvalid() raise NameInvalid()
@ -145,7 +145,7 @@ def _write_manifest(namespace_name, repo_name, manifest):
raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'}) raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})
# Ensure all the blobs in the manifest exist. # Ensure all the blobs in the manifest exist.
storage_map = v2.lookup_blobs_by_digest(namespace_name, repo_name, manifest.checksums) storage_map = model.lookup_blobs_by_digest(namespace_name, repo_name, manifest.checksums)
for layer in manifest.layers: for layer in manifest.layers:
digest_str = str(layer.digest) digest_str = str(layer.digest)
if digest_str not in storage_map: if digest_str not in storage_map:
@ -154,13 +154,13 @@ def _write_manifest(namespace_name, repo_name, manifest):
# Lookup all the images and their parent images (if any) inside the manifest. # Lookup all the images and their parent images (if any) inside the manifest.
# This will let us know which v1 images we need to synthesize and which ones are invalid. # This will let us know which v1 images we need to synthesize and which ones are invalid.
all_image_ids = list(manifest.parent_image_ids | manifest.image_ids) all_image_ids = list(manifest.parent_image_ids | manifest.image_ids)
images_map = v2.get_docker_v1_metadata_by_image_id(namespace_name, repo_name, all_image_ids) images_map = model.get_docker_v1_metadata_by_image_id(namespace_name, repo_name, all_image_ids)
# Rewrite any v1 image IDs that do not match the checksum in the database. # Rewrite any v1 image IDs that do not match the checksum in the database.
try: try:
rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map)) rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map))
for rewritten_image in rewritten_images: for rewritten_image in rewritten_images:
v1_metadata = v2.synthesize_v1_image( model.synthesize_v1_image(
repo, repo,
storage_map[rewritten_image.content_checksum], storage_map[rewritten_image.content_checksum],
rewritten_image.image_id, rewritten_image.image_id,
@ -175,8 +175,8 @@ def _write_manifest(namespace_name, repo_name, manifest):
# Store the manifest pointing to the tag. # Store the manifest pointing to the tag.
leaf_layer_id = rewritten_images[-1].image_id leaf_layer_id = rewritten_images[-1].image_id
v2.save_manifest(namespace_name, repo_name, manifest.tag, leaf_layer_id, manifest.digest, model.save_manifest(namespace_name, repo_name, manifest.tag, leaf_layer_id, manifest.digest,
manifest.bytes) manifest.bytes)
# Queue all blob manifests for replication. # Queue all blob manifests for replication.
# TODO(jschorr): Find a way to optimize this insertion. # TODO(jschorr): Find a way to optimize this insertion.
@ -213,7 +213,7 @@ def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
Note: there is no equivalent method for deleting by tag name because it is Note: there is no equivalent method for deleting by tag name because it is
forbidden by the spec. forbidden by the spec.
""" """
tags = v2.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref) tags = model.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref)
if not tags: if not tags:
raise ManifestUnknown() raise ManifestUnknown()
@ -225,9 +225,9 @@ def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
def _generate_and_store_manifest(namespace_name, repo_name, tag_name): def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
# Find the v1 metadata for this image and its parents. # Find the v1 metadata for this image and its parents.
v1_metadata = v2.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name) v1_metadata = model.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name)
parents_v1_metadata = v2.get_parents_docker_v1_metadata(namespace_name, repo_name, parents_v1_metadata = model.get_parents_docker_v1_metadata(namespace_name, repo_name,
v1_metadata.image_id) v1_metadata.image_id)
# If the manifest is being generated under the library namespace, then we make its namespace # If the manifest is being generated under the library namespace, then we make its namespace
# empty. # empty.
@ -248,6 +248,6 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
manifest = builder.build(docker_v2_signing_key) manifest = builder.build(docker_v2_signing_key)
# Write the manifest to the DB. # Write the manifest to the DB.
v2.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest,
manifest.bytes) manifest.bytes)
return manifest return manifest

6
endpoints/v2/tag.py
View file

@ -5,7 +5,7 @@ from endpoints.common import parse_repository_name
from endpoints.v2 import v2_bp, require_repo_read, paginate from endpoints.v2 import v2_bp, require_repo_read, paginate
from endpoints.v2.errors import NameUnknown from endpoints.v2.errors import NameUnknown
from endpoints.decorators import anon_protect from endpoints.decorators import anon_protect
from data.interfaces import v2 from data.interfaces.v2 import PreOCIModel as model
@v2_bp.route('/<repopath:repository>/tags/list', methods=['GET']) @v2_bp.route('/<repopath:repository>/tags/list', methods=['GET'])
@parse_repository_name() @parse_repository_name()
@ -14,11 +14,11 @@ from data.interfaces import v2
@anon_protect @anon_protect
@paginate() @paginate()
def list_all_tags(namespace_name, repo_name, limit, offset, pagination_callback): def list_all_tags(namespace_name, repo_name, limit, offset, pagination_callback):
repo = v2.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if repo is None: if repo is None:
raise NameUnknown() raise NameUnknown()
tags = v2.repository_tags(namespace_name, repo_name, limit, offset) tags = model.repository_tags(namespace_name, repo_name, limit, offset)
response = jsonify({ response = jsonify({
'name': '{0}/{1}'.format(namespace_name, repo_name), 'name': '{0}/{1}'.format(namespace_name, repo_name),
'tags': [tag.name for tag in tags], 'tags': [tag.name for tag in tags],

8
endpoints/v2/v2auth.py
View file

@ -11,7 +11,7 @@ from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermissi
CreateRepositoryPermission) CreateRepositoryPermission)
from endpoints.v2 import v2_bp from endpoints.v2 import v2_bp
from endpoints.decorators import anon_protect from endpoints.decorators import anon_protect
from data.interfaces import v2 from data.interfaces.v2 import PreOCIModel as model
from util.cache import no_cache from util.cache import no_cache
from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX
from util.security.registry_jwt import generate_bearer_token, build_context_and_subject from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
@ -96,7 +96,7 @@ def generate_registry_jwt():
if user is not None or token is not None: if user is not None or token is not None:
# Lookup the repository. If it exists, make sure the entity has modify # Lookup the repository. If it exists, make sure the entity has modify
# permission. Otherwise, make sure the entity has create permission. # permission. Otherwise, make sure the entity has create permission.
repo = v2.get_repository(namespace, reponame) repo = model.get_repository(namespace, reponame)
if repo: if repo:
if ModifyRepositoryPermission(namespace, reponame).can(): if ModifyRepositoryPermission(namespace, reponame).can():
final_actions.append('push') final_actions.append('push')
@ -105,7 +105,7 @@ def generate_registry_jwt():
else: else:
if CreateRepositoryPermission(namespace).can() and user is not None: if CreateRepositoryPermission(namespace).can() and user is not None:
logger.debug('Creating repository: %s/%s', namespace, reponame) logger.debug('Creating repository: %s/%s', namespace, reponame)
v2.create_repository(namespace, reponame, user) model.create_repository(namespace, reponame, user)
final_actions.append('push') final_actions.append('push')
else: else:
logger.debug('No permission to create repository %s/%s', namespace, reponame) logger.debug('No permission to create repository %s/%s', namespace, reponame)
@ -113,7 +113,7 @@ def generate_registry_jwt():
if 'pull' in actions: if 'pull' in actions:
# Grant pull if the user can read the repo or it is public. # Grant pull if the user can read the repo or it is public.
if (ReadRepositoryPermission(namespace, reponame).can() or if (ReadRepositoryPermission(namespace, reponame).can() or
v2.repository_is_public(namespace, reponame)): model.repository_is_public(namespace, reponame)):
final_actions.append('pull') final_actions.append('pull')
else: else:
logger.debug('No permission to pull repository %s/%s', namespace, reponame) logger.debug('No permission to pull repository %s/%s', namespace, reponame)

111
image/__init__.py
View file

@ -1,111 +0,0 @@
import tarfile
from collections import namedtuple
from namedlist import namedlist
from util.registry.gzipwrap import GzipWrap
class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
"""
ManifestJSON represents a Manifest of any format.
"""
class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
"""
RepositoryReference represents a reference to a Repository, without its full metadata.
"""
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
'is_public'])):
"""
Repository represents a collection of tags.
"""
class Tag(namedtuple('Tag', ['name', 'repository'])):
"""
Tag represents a user-facing alias for referencing a set of Manifests.
"""
class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count',
'chunk_count', 'sha_state', 'location_name',
'storage_metadata', 'piece_sha_state', 'piece_hashes',
'repo_namespace_name', 'repo_name'])):
"""
BlobUpload represents the current state of an Blob being uploaded.
"""
class Blob(namedtuple('Blob', ['uuid', 'digest', 'size', 'locations'])):
"""
Blob represents an opaque binary blob saved to the storage system.
"""
class TarImageFormatter(object):
"""
Base class for classes which produce a tar containing image and layer data.
"""
def build_stream(self, namespace, repository, tag, synthetic_image_id, layer_json,
get_image_iterator, get_layer_iterator, get_image_json):
"""
Builds and streams a synthetic .tar.gz that represents the formatted tar created by this class's
implementation.
"""
return GzipWrap(self.stream_generator(namespace, repository, tag,
synthetic_image_id, layer_json,
get_image_iterator, get_layer_iterator,
get_image_json))
def stream_generator(self, namespace, repository, tag, synthetic_image_id,
layer_json, get_image_iterator, get_layer_iterator, get_image_json):
raise NotImplementedError
def tar_file(self, name, contents, mtime=None):
"""
Returns the tar binary representation for a file with the given name and file contents.
"""
length = len(contents)
tar_data = self.tar_file_header(name, length, mtime=mtime)
tar_data += contents
tar_data += self.tar_file_padding(length)
return tar_data
def tar_file_padding(self, length):
"""
Returns tar file padding for file data of the given length.
"""
if length % 512 != 0:
return '\0' * (512 - (length % 512))
return ''
def tar_file_header(self, name, file_size, mtime=None):
"""
Returns tar file header data for a file with the given name and size.
"""
info = tarfile.TarInfo(name=name)
info.type = tarfile.REGTYPE
info.size = file_size
if mtime is not None:
info.mtime = mtime
return info.tobuf()
def tar_folder(self, name, mtime=None):
"""
Returns tar file header data for a folder with the given name.
"""
info = tarfile.TarInfo(name=name)
info.type = tarfile.DIRTYPE
if mtime is not None:
info.mtime = mtime
# allow the directory to be readable by non-root users
info.mode = 0755
return info.tobuf()

2
image/appc/__init__.py
View file

@ -6,7 +6,7 @@ from uuid import uuid4
from app import app from app import app
from util.registry.streamlayerformat import StreamLayerMerger from util.registry.streamlayerformat import StreamLayerMerger
from image import TarImageFormatter from image.common import TarImageFormatter
ACNAME_REGEX = re.compile(r'[^a-z-]+') ACNAME_REGEX = re.compile(r'[^a-z-]+')

68
image/common.py Normal file
View file

@ -0,0 +1,68 @@
import tarfile
from util.registry.gzipwrap import GzipWrap
class TarImageFormatter(object):
"""
Base class for classes which produce a tar containing image and layer data.
"""
def build_stream(self, namespace, repository, tag, synthetic_image_id, layer_json,
get_image_iterator, get_layer_iterator, get_image_json):
"""
Builds and streams a synthetic .tar.gz that represents the formatted tar created by this class's
implementation.
"""
return GzipWrap(self.stream_generator(namespace, repository, tag,
synthetic_image_id, layer_json,
get_image_iterator, get_layer_iterator,
get_image_json))
def stream_generator(self, namespace, repository, tag, synthetic_image_id,
layer_json, get_image_iterator, get_layer_iterator, get_image_json):
raise NotImplementedError
def tar_file(self, name, contents, mtime=None):
"""
Returns the tar binary representation for a file with the given name and file contents.
"""
length = len(contents)
tar_data = self.tar_file_header(name, length, mtime=mtime)
tar_data += contents
tar_data += self.tar_file_padding(length)
return tar_data
def tar_file_padding(self, length):
"""
Returns tar file padding for file data of the given length.
"""
if length % 512 != 0:
return '\0' * (512 - (length % 512))
return ''
def tar_file_header(self, name, file_size, mtime=None):
"""
Returns tar file header data for a file with the given name and size.
"""
info = tarfile.TarInfo(name=name)
info.type = tarfile.REGTYPE
info.size = file_size
if mtime is not None:
info.mtime = mtime
return info.tobuf()
def tar_folder(self, name, mtime=None):
"""
Returns tar file header data for a folder with the given name.
"""
info = tarfile.TarInfo(name=name)
info.type = tarfile.DIRTYPE
if mtime is not None:
info.mtime = mtime
# allow the directory to be readable by non-root users
info.mode = 0755
return info.tobuf()

2
image/docker/squashed.py
View file

@ -4,7 +4,7 @@ import math
import calendar import calendar
from app import app from app import app
from image import TarImageFormatter from image.common import TarImageFormatter
from util.registry.gzipwrap import GZIP_BUFFER_SIZE from util.registry.gzipwrap import GZIP_BUFFER_SIZE
from util.registry.streamlayerformat import StreamLayerMerger from util.registry.streamlayerformat import StreamLayerMerger