vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Optimize the new registry data model to avoid unnecessary queries

Browse source
This commit is contained in:
Joseph Schorr 2018-08-24 11:25:24 -04:00
parent 46edebe6b0
commit a96c5a7f64
4 changed files with 41 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

6
data/model/image.py
View file

@ -357,7 +357,11 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
def get_image(repo, docker_image_id): def get_image(repo, docker_image_id):
try: try:
return Image.get(Image.docker_image_id == docker_image_id, Image.repository == repo) return (Image
.select(Image, ImageStorage)
.join(ImageStorage)
.where(Image.docker_image_id == docker_image_id, Image.repository == repo)
.get())
except Image.DoesNotExist: except Image.DoesNotExist:
return None return None

10
data/model/tag.py
View file

@ -208,8 +208,9 @@ def list_active_repo_tags(repo):
and (if present), their manifest. and (if present), their manifest.
""" """
query = _tag_alive(RepositoryTag query = _tag_alive(RepositoryTag
.select(RepositoryTag, Image, TagManifest.digest) .select(RepositoryTag, Image, ImageStorage, TagManifest.digest)
.join(Image) .join(Image)
.join(ImageStorage)
.where(RepositoryTag.repository == repo, RepositoryTag.hidden == False) .where(RepositoryTag.repository == repo, RepositoryTag.hidden == False)
.switch(RepositoryTag) .switch(RepositoryTag)
.join(TagManifest, JOIN.LEFT_OUTER)) .join(TagManifest, JOIN.LEFT_OUTER))
@ -470,8 +471,9 @@ def get_tag_image(namespace_name, repository_name, tag_name, include_storage=Fal
def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None): def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None):
query = (RepositoryTag query = (RepositoryTag
.select(RepositoryTag, Image) .select(RepositoryTag, Image, ImageStorage)
.join(Image) .join(Image)
.join(ImageStorage)
.switch(RepositoryTag) .switch(RepositoryTag)
.where(RepositoryTag.repository == repo_obj) .where(RepositoryTag.repository == repo_obj)
.where(RepositoryTag.hidden == False) .where(RepositoryTag.hidden == False)
@ -591,7 +593,9 @@ def get_active_tag(namespace, repo_name, tag_name):
def get_active_tag_for_repo(repo, tag_name): def get_active_tag_for_repo(repo, tag_name):
try: try:
return _tag_alive(RepositoryTag return _tag_alive(RepositoryTag
.select() .select(RepositoryTag, Image, ImageStorage)
.join(Image)
.join(ImageStorage)
.where(RepositoryTag.name == tag_name, .where(RepositoryTag.name == tag_name,
RepositoryTag.repository == repo)).get() RepositoryTag.repository == repo)).get()
except RepositoryTag.DoesNotExist: except RepositoryTag.DoesNotExist:

3
data/registry_model/datatypes.py
View file

@ -70,7 +70,7 @@ class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])):
class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'comment', 'command', class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'comment', 'command',
'image_size', 'uploading'])): 'image_size', 'aggregate_size', 'uploading'])):
""" LegacyImage represents a Docker V1-style image found in a repository. """ """ LegacyImage represents a Docker V1-style image found in a repository. """
@classmethod @classmethod
def for_image(cls, image, images_map=None, tags_map=None): def for_image(cls, image, images_map=None, tags_map=None):
@ -85,6 +85,7 @@ class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'commen
comment=image.comment, comment=image.comment,
command=image.command, command=image.command,
image_size=image.storage.image_size, image_size=image.storage.image_size,
aggregate_size=image.aggregate_size,
uploading=image.storage.uploading) uploading=image.storage.uploading)
@property @property

41
data/registry_model/test/test_pre_oci_model.py
View file

@ -2,9 +2,12 @@ from datetime import datetime, timedelta
import pytest import pytest
from playhouse.test_utils import assert_query_count
from data import model from data import model
from data.registry_model.registry_pre_oci_model import PreOCIModel from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.datatypes import RepositoryReference from data.registry_model.datatypes import RepositoryReference
from test.fixtures import * from test.fixtures import *
@pytest.fixture() @pytest.fixture()
@ -95,8 +98,11 @@ def test_legacy_images(repo_namespace, repo_name, pre_oci_model):
found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True) include_parents=True)
assert found_image.docker_image_id == image.docker_image_id with assert_query_count(4 if found_image.parents else 3):
assert found_image.parents == image.parents found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True)
assert found_image.docker_image_id == image.docker_image_id
assert found_image.parents == image.parents
# Check that the tags list can be retrieved. # Check that the tags list can be retrieved.
assert image.tags is not None assert image.tags is not None
@ -157,26 +163,32 @@ def test_manifest_labels(pre_oci_model):
]) ])
def test_repository_tags(repo_namespace, repo_name, pre_oci_model): def test_repository_tags(repo_namespace, repo_name, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref)
assert len(tags) with assert_query_count(1):
tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert len(tags)
for tag in tags: for tag in tags:
found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) with assert_query_count(2):
assert found_tag == tag found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag == tag
if found_tag.legacy_image is None: if found_tag.legacy_image is None:
continue continue
found_image = pre_oci_model.get_legacy_image(repository_ref, with assert_query_count(2):
found_tag.legacy_image.docker_image_id) found_image = pre_oci_model.get_legacy_image(repository_ref,
assert found_image == found_tag.legacy_image found_tag.legacy_image.docker_image_id)
assert found_image == found_tag.legacy_image
def test_repository_tag_history(pre_oci_model): def test_repository_tag_history(pre_oci_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'history') repository_ref = pre_oci_model.lookup_repository('devtable', 'history')
history, has_more = pre_oci_model.list_repository_tag_history(repository_ref)
assert not has_more with assert_query_count(2):
assert len(history) == 2 history, has_more = pre_oci_model.list_repository_tag_history(repository_ref)
assert not has_more
assert len(history) == 2
@pytest.mark.parametrize('repo_namespace, repo_name', [ @pytest.mark.parametrize('repo_namespace, repo_name', [
@ -199,8 +211,9 @@ def test_delete_tags(repo_namespace, repo_name, pre_oci_model):
assert pre_oci_model.delete_tag(repository_ref, tag.name) assert pre_oci_model.delete_tag(repository_ref, tag.name)
# Make sure the tag is no longer found. # Make sure the tag is no longer found.
found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) with assert_query_count(1):
assert found_tag is None found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag is None
# Ensure all tags have been deleted. # Ensure all tags have been deleted.
tags = pre_oci_model.list_repository_tags(repository_ref) tags = pre_oci_model.list_repository_tags(repository_ref)