b5bb76cdea
Previous to this change, repositories were looked up unfiltered in six different queries, and then filtered using the permissions model, which issued a query per repository found, making search incredibly slow. Instead, we now lookup a chunk of repositories unfiltered and then filter them via a single query to the database. By layering the filtering on top of the lookup, each as queries, we can minimize the number of queries necessary, without (at the same time) using a super expensive join. Other changes: - Remove the 5 page pre-lookup on V1 search and simply return that there is one more page available, until there isn't. While technically not correct, it is much more efficient, and no one should be using pagination with V1 search anyway. - Remove the lookup for repos without entries in the RAC table. Instead, we now add a new RAC entry when the repository is created for *the day before*, with count 0, so that it is immediately searchable - Remove lookup of results with a matching namespace; these aren't very relevant anyway, and it overly complicates sorting
402 lines
14 KiB
Python
402 lines
14 KiB
Python
from abc import ABCMeta, abstractmethod
|
|
from collections import namedtuple
|
|
|
|
from six import add_metaclass
|
|
|
|
from app import app, storage as store
|
|
from data import model
|
|
from data.model import db_transaction
|
|
from util.morecollections import AttrDict
|
|
|
|
|
|
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
|
|
'is_public'])):
|
|
"""
|
|
Repository represents a namespaced collection of tags.
|
|
"""
|
|
|
|
|
|
@add_metaclass(ABCMeta)
|
|
class DockerRegistryV1DataInterface(object):
|
|
"""
|
|
Interface that represents all data store interactions required by a Docker Registry v1.
|
|
"""
|
|
|
|
@abstractmethod
|
|
def placement_locations_and_path_docker_v1(self, namespace_name, repo_name, image_id):
|
|
"""
|
|
Returns all the placements for the image with the given V1 Docker ID, found under the given
|
|
repository or None if no image was found.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def docker_v1_metadata(self, namespace_name, repo_name, image_id):
|
|
"""
|
|
Returns various pieces of metadata associated with an image with the given V1 Docker ID,
|
|
including the checksum and its V1 JSON metadata.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def update_docker_v1_metadata(self, namespace_name, repo_name, image_id, created_date_str,
|
|
comment, command, compat_json, parent_image_id=None):
|
|
"""
|
|
Updates various pieces of V1 metadata associated with a particular image.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def storage_exists(self, namespace_name, repo_name, image_id):
|
|
"""
|
|
Returns whether storage already exists for the image with the V1 Docker ID under the given
|
|
repository.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def store_docker_v1_checksums(self, namespace_name, repo_name, image_id, checksum,
|
|
content_checksum):
|
|
"""
|
|
Stores the various V1 checksums for the image with the V1 Docker ID.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def is_image_uploading(self, namespace_name, repo_name, image_id):
|
|
"""
|
|
Returns whether the image with the V1 Docker ID is currently marked as uploading.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def update_image_uploading(self, namespace_name, repo_name, image_id, is_uploading):
|
|
"""
|
|
Marks the image with the V1 Docker ID with the given uploading status.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def update_image_sizes(self, namespace_name, repo_name, image_id, size, uncompressed_size):
|
|
"""
|
|
Updates the sizing information for the image with the given V1 Docker ID.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_image_size(self, namespace_name, repo_name, image_id):
|
|
"""
|
|
Returns the wire size of the image with the given Docker V1 ID.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_bittorrent_pieces(self, namespace_name, repo_name, image_id, pieces_bytes):
|
|
"""
|
|
Saves the BitTorrent piece hashes for the image with the given Docker V1 ID.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def image_ancestry(self, namespace_name, repo_name, image_id):
|
|
"""
|
|
Returns a list containing the full ancestry of Docker V1 IDs, in order, for the image with the
|
|
given Docker V1 ID.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def repository_exists(self, namespace_name, repo_name):
|
|
"""
|
|
Returns whether the repository with the given name and namespace exists.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_or_link_image(self, username, namespace_name, repo_name, image_id, storage_location):
|
|
"""
|
|
Adds the given image to the given repository, by either linking to an existing image visible to
|
|
the user with the given username, or creating a new one if no existing image matches.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_temp_hidden_tag(self, namespace_name, repo_name, image_id, expiration):
|
|
"""
|
|
Creates a hidden tag under the matching namespace pointing to the image with the given V1 Docker
|
|
ID.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def list_tags(self, namespace_name, repo_name):
|
|
"""
|
|
Returns all the tags defined in the repository with the given namespace and name.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_or_update_tag(self, namespace_name, repo_name, image_id, tag_name):
|
|
"""
|
|
Creates or updates a tag under the matching repository to point to the image with the given
|
|
Docker V1 ID.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def find_image_id_by_tag(self, namespace_name, repo_name, tag_name):
|
|
"""
|
|
Returns the Docker V1 image ID for the HEAD image for the tag with the given name under the
|
|
matching repository, or None if none.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def delete_tag(self, namespace_name, repo_name, tag_name):
|
|
"""
|
|
Deletes the given tag from the given repository.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def load_token(self, token):
|
|
"""
|
|
Loads the data associated with the given (deprecated) access token, and, if
|
|
found returns True.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def verify_robot(self, username, token):
|
|
"""
|
|
Returns True if the given robot username and token match an existing robot
|
|
account.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def change_user_password(self, user, new_password):
|
|
"""
|
|
Changes the password associated with the given user.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_repository(self, namespace_name, repo_name):
|
|
"""
|
|
Returns the repository with the given name under the given namespace or None
|
|
if none.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def create_repository(self, namespace_name, repo_name, user=None):
|
|
"""
|
|
Creates a new repository under the given namespace with the given name, for
|
|
the given user.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def repository_is_public(self, namespace_name, repo_name):
|
|
"""
|
|
Returns whether the repository with the given name under the given namespace
|
|
is public. If no matching repository was found, returns False.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def validate_oauth_token(self, token):
|
|
""" Returns whether the given OAuth token validates. """
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_sorted_matching_repositories(self, search_term, filter_username=None, offset=0, limit=25):
|
|
"""
|
|
Returns a sorted list of repositories matching the given search term.
|
|
"""
|
|
pass
|
|
|
|
|
|
class PreOCIModel(DockerRegistryV1DataInterface):
|
|
"""
|
|
PreOCIModel implements the data model for the v1 Docker Registry protocol using a database schema
|
|
before it was changed to support the OCI specification.
|
|
"""
|
|
def placement_locations_and_path_docker_v1(self, namespace_name, repo_name, image_id):
|
|
repo_image = model.image.get_repo_image_extended(namespace_name, repo_name, image_id)
|
|
if not repo_image or repo_image.storage is None:
|
|
return None, None
|
|
return repo_image.storage.locations, model.storage.get_layer_path(repo_image.storage)
|
|
|
|
def docker_v1_metadata(self, namespace_name, repo_name, image_id):
|
|
repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id)
|
|
if repo_image is None:
|
|
return None
|
|
|
|
return AttrDict({
|
|
'namespace_name': namespace_name,
|
|
'repo_name': repo_name,
|
|
'image_id': image_id,
|
|
'checksum': repo_image.v1_checksum,
|
|
'compat_json': repo_image.v1_json_metadata,
|
|
})
|
|
|
|
def update_docker_v1_metadata(self, namespace_name, repo_name, image_id, created_date_str,
|
|
comment, command, compat_json, parent_image_id=None):
|
|
parent_image = None
|
|
if parent_image_id is not None:
|
|
parent_image = model.image.get_repo_image(namespace_name, repo_name, parent_image_id)
|
|
|
|
model.image.set_image_metadata(image_id, namespace_name, repo_name, created_date_str, comment,
|
|
command, compat_json, parent=parent_image)
|
|
|
|
def storage_exists(self, namespace_name, repo_name, image_id):
|
|
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
|
|
if repo_image is None or repo_image.storage is None:
|
|
return False
|
|
|
|
if repo_image.storage.uploading:
|
|
return False
|
|
|
|
layer_path = model.storage.get_layer_path(repo_image.storage)
|
|
return store.exists(repo_image.storage.locations, layer_path)
|
|
|
|
def store_docker_v1_checksums(self, namespace_name, repo_name, image_id, checksum,
|
|
content_checksum):
|
|
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
|
|
if repo_image is None or repo_image.storage is None:
|
|
return
|
|
|
|
with db_transaction():
|
|
repo_image.storage.content_checksum = content_checksum
|
|
repo_image.v1_checksum = checksum
|
|
repo_image.storage.save()
|
|
repo_image.save()
|
|
|
|
def is_image_uploading(self, namespace_name, repo_name, image_id):
|
|
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
|
|
if repo_image is None or repo_image.storage is None:
|
|
return False
|
|
return repo_image.storage.uploading
|
|
|
|
def update_image_uploading(self, namespace_name, repo_name, image_id, is_uploading):
|
|
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
|
|
if repo_image is None or repo_image.storage is None:
|
|
return
|
|
|
|
repo_image.storage.uploading = is_uploading
|
|
repo_image.storage.save()
|
|
return repo_image.storage
|
|
|
|
def update_image_sizes(self, namespace_name, repo_name, image_id, size, uncompressed_size):
|
|
model.storage.set_image_storage_metadata(image_id, namespace_name, repo_name, size,
|
|
uncompressed_size)
|
|
|
|
def get_image_size(self, namespace_name, repo_name, image_id):
|
|
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
|
|
if repo_image is None or repo_image.storage is None:
|
|
return None
|
|
return repo_image.storage.image_size
|
|
|
|
def create_bittorrent_pieces(self, namespace_name, repo_name, image_id, pieces_bytes):
|
|
repo_image = model.image.get_repo_image_and_storage(namespace_name, repo_name, image_id)
|
|
if repo_image is None or repo_image.storage is None:
|
|
return
|
|
|
|
model.storage.save_torrent_info(repo_image.storage, app.config['BITTORRENT_PIECE_SIZE'],
|
|
pieces_bytes)
|
|
|
|
def image_ancestry(self, namespace_name, repo_name, image_id):
|
|
try:
|
|
image = model.image.get_image_by_id(namespace_name, repo_name, image_id)
|
|
except model.InvalidImageException:
|
|
return None
|
|
|
|
parents = model.image.get_parent_images(namespace_name, repo_name, image)
|
|
ancestry_docker_ids = [image.docker_image_id]
|
|
ancestry_docker_ids.extend([parent.docker_image_id for parent in parents])
|
|
return ancestry_docker_ids
|
|
|
|
def repository_exists(self, namespace_name, repo_name):
|
|
repo = model.repository.get_repository(namespace_name, repo_name)
|
|
return repo is not None
|
|
|
|
def create_or_link_image(self, username, namespace_name, repo_name, image_id, storage_location):
|
|
repo = model.repository.get_repository(namespace_name, repo_name)
|
|
model.image.find_create_or_link_image(image_id, repo, username, {}, storage_location)
|
|
|
|
def create_temp_hidden_tag(self, namespace_name, repo_name, image_id, expiration):
|
|
repo_image = model.image.get_repo_image(namespace_name, repo_name, image_id)
|
|
if repo_image is None:
|
|
return
|
|
|
|
repo = repo_image.repository
|
|
model.tag.create_temporary_hidden_tag(repo, repo_image, expiration)
|
|
|
|
def list_tags(self, namespace_name, repo_name):
|
|
return model.tag.list_repository_tags(namespace_name, repo_name)
|
|
|
|
def create_or_update_tag(self, namespace_name, repo_name, image_id, tag_name):
|
|
model.tag.create_or_update_tag(namespace_name, repo_name, tag_name, image_id)
|
|
|
|
def find_image_id_by_tag(self, namespace_name, repo_name, tag_name):
|
|
try:
|
|
tag_image = model.tag.get_tag_image(namespace_name, repo_name, tag_name)
|
|
except model.DataModelException:
|
|
return None
|
|
|
|
return tag_image.docker_image_id
|
|
|
|
def delete_tag(self, namespace_name, repo_name, tag_name):
|
|
model.tag.delete_tag(namespace_name, repo_name, tag_name)
|
|
|
|
def load_token(self, token):
|
|
try:
|
|
model.token.load_token_data(token)
|
|
return True
|
|
except model.InvalidTokenException:
|
|
return False
|
|
|
|
def verify_robot(self, username, token):
|
|
try:
|
|
return bool(model.user.verify_robot(username, token))
|
|
except model.InvalidRobotException:
|
|
return False
|
|
|
|
def change_user_password(self, user, new_password):
|
|
model.user.change_password(user, new_password)
|
|
|
|
def get_repository(self, namespace_name, repo_name):
|
|
repo = model.repository.get_repository(namespace_name, repo_name)
|
|
if repo is None:
|
|
return None
|
|
return _repository_for_repo(repo)
|
|
|
|
def create_repository(self, namespace_name, repo_name, user=None):
|
|
model.repository.create_repository(namespace_name, repo_name, user)
|
|
|
|
def repository_is_public(self, namespace_name, repo_name):
|
|
return model.repository.repository_is_public(namespace_name, repo_name)
|
|
|
|
def validate_oauth_token(self, token):
|
|
return bool(model.oauth.validate_access_token(token))
|
|
|
|
def get_sorted_matching_repositories(self, search_term, filter_username=None, offset=0, limit=25):
|
|
repos = model.repository.get_filtered_matching_repositories(search_term, filter_username,
|
|
offset, limit)
|
|
return [_repository_for_repo(repo) for repo in repos]
|
|
|
|
|
|
def _repository_for_repo(repo):
|
|
""" Returns a Repository object representing the Pre-OCI data model instance of a repository. """
|
|
return Repository(
|
|
id=repo.id,
|
|
name=repo.name,
|
|
namespace_name=repo.namespace_user.username,
|
|
description=repo.description,
|
|
is_public=model.repository.is_repository_public(repo)
|
|
)
|
|
|
|
|
|
pre_oci_model = PreOCIModel()
|