vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add sharing to the images lookup in get_matching_tags_for_images

Browse source 
Should prevent an issue if we ever get a request with thousands and thousands of images where we would exceed the DB's max packet size
This commit is contained in:
Joseph Schorr 2017-05-02 15:50:31 -04:00
parent 98fcae753b
commit d77463cc8b
2 changed files with 72 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

80
data/model/test/test_tag.py
View file

@ -1,3 +1,7 @@
import pytest
from mock import patch
from data.database import Image, RepositoryTag, ImageStorage, Repository
from data.model.repository import create_repository
from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag,
@ -14,42 +18,62 @@ def _get_expected_tags(image):
.where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id))))
return set([tag.id for tag in _tag_alive(expected_query)])
def test_get_matching_tags(initialized_db):
# Test for every image in the test database.
for image in Image.select(Image, ImageStorage).join(ImageStorage):
matching_query = get_matching_tags(image.docker_image_id, image.storage.uuid)
matching_tags = set([tag.id for tag in matching_query])
expected_tags = _get_expected_tags(image)
assert matching_tags == expected_tags, "mismatch for image %s" % image.id
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
(1, 1),
(10, 10),
(100, 500),
])
def test_get_matching_tags(max_subqueries, max_image_lookup_count, initialized_db):
with patch('data.model.tag._MAX_SUB_QUERIES', max_subqueries):
with patch('data.model.tag._MAX_IMAGE_LOOKUP_COUNT', max_image_lookup_count):
# Test for every image in the test database.
for image in Image.select(Image, ImageStorage).join(ImageStorage):
matching_query = get_matching_tags(image.docker_image_id, image.storage.uuid)
matching_tags = set([tag.id for tag in matching_query])
expected_tags = _get_expected_tags(image)
assert matching_tags == expected_tags, "mismatch for image %s" % image.id
def test_get_matching_tag_ids_for_images(initialized_db):
# Try for various sets of the first N images.
for count in [5, 10, 15]:
pairs = []
expected_tags_ids = set()
for image in Image.select(Image, ImageStorage).join(ImageStorage):
if len(pairs) >= count:
break
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
(1, 1),
(10, 10),
(100, 500),
])
def test_get_matching_tag_ids_for_images(max_subqueries, max_image_lookup_count, initialized_db):
with patch('data.model.tag._MAX_SUB_QUERIES', max_subqueries):
with patch('data.model.tag._MAX_IMAGE_LOOKUP_COUNT', max_image_lookup_count):
# Try for various sets of the first N images.
for count in [5, 10, 15]:
pairs = []
expected_tags_ids = set()
for image in Image.select(Image, ImageStorage).join(ImageStorage):
if len(pairs) >= count:
break
pairs.append((image.docker_image_id, image.storage.uuid))
expected_tags_ids.update(_get_expected_tags(image))
pairs.append((image.docker_image_id, image.storage.uuid))
expected_tags_ids.update(_get_expected_tags(image))
matching_tags_ids = set([tag.id for tag in get_matching_tags_for_images(pairs)])
assert matching_tags_ids == expected_tags_ids
matching_tags_ids = set([tag.id for tag in get_matching_tags_for_images(pairs)])
assert matching_tags_ids == expected_tags_ids
def test_get_matching_tag_ids_for_all_images(initialized_db):
pairs = []
for image in Image.select(Image, ImageStorage).join(ImageStorage):
pairs.append((image.docker_image_id, image.storage.uuid))
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
(1, 1),
(10, 10),
(100, 500),
])
def test_get_matching_tag_ids_for_all_images(max_subqueries, max_image_lookup_count, initialized_db):
with patch('data.model.tag._MAX_SUB_QUERIES', max_subqueries):
with patch('data.model.tag._MAX_IMAGE_LOOKUP_COUNT', max_image_lookup_count):
pairs = []
for image in Image.select(Image, ImageStorage).join(ImageStorage):
pairs.append((image.docker_image_id, image.storage.uuid))
expected_tags_ids = set([tag.id for tag in _tag_alive(RepositoryTag.select())])
matching_tags_ids = set([tag.id for tag in get_matching_tags_for_images(pairs)])
expected_tags_ids = set([tag.id for tag in _tag_alive(RepositoryTag.select())])
matching_tags_ids = set([tag.id for tag in get_matching_tags_for_images(pairs)])
# Ensure every alive tag was found.
assert matching_tags_ids == expected_tags_ids
# Ensure every alive tag was found.
assert matching_tags_ids == expected_tags_ids
def test_get_matching_tag_ids_images_filtered(initialized_db):