vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add sharing to the images lookup in get_matching_tags_for_images

Browse source 
Should prevent an issue if we ever get a request with thousands and thousands of images where we would exceed the DB's max packet size
This commit is contained in:
Joseph Schorr 2017-05-02 15:50:31 -04:00
parent 98fcae753b
commit d77463cc8b
2 changed files with 72 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

30
data/model/tag.py
View file

@ -54,6 +54,7 @@ def _tag_alive(query, now_ts=None):
_MAX_SUB_QUERIES = 100
_MAX_IMAGE_LOOKUP_COUNT = 500
def get_matching_tags_for_images(image_pairs, filter_query=None, selections=None):
""" Returns all tags that contain the images with the given docker_image_id and storage_uuid,
@ -61,27 +62,36 @@ def get_matching_tags_for_images(image_pairs, filter_query=None, selections=None
if not image_pairs:
return []
image_pairs = set(image_pairs)
image_pairs_set = set(image_pairs)
# Find all possible matching image+storages.
ids = [image_pair[0] for image_pair in image_pairs]
uuids = [image_pair[1] for image_pair in image_pairs]
images_query = (Image
.select(Image.id, Image.docker_image_id, Image.ancestors, ImageStorage.uuid)
.join(ImageStorage)
.where(Image.docker_image_id << ids, ImageStorage.uuid << uuids))
images = []
while image_pairs:
image_pairs_slice = image_pairs[0:_MAX_IMAGE_LOOKUP_COUNT]
ids = [pair[0] for pair in image_pairs_slice]
uuids = [pair[1] for pair in image_pairs_slice]
images_query = (Image
.select(Image.id, Image.docker_image_id, Image.ancestors, ImageStorage.uuid)
.join(ImageStorage)
.where(Image.docker_image_id << ids, ImageStorage.uuid << uuids))
images.extend(list(images_query))
image_pairs = image_pairs[_MAX_IMAGE_LOOKUP_COUNT:]
# Filter down to those images actually in the pairs set and build the set of queries to run.
individual_image_queries = []
for img in images_query:
for img in images:
# Make sure the actual image was requested.
pair = (img.docker_image_id, img.storage.uuid)
if pair not in image_pairs:
if pair not in image_pairs_set:
continue
# Remove the pair so we don't try it again.
image_pairs.remove(pair)
image_pairs_set.remove(pair)
ancestors_str = '%s%s/%%' % (img.ancestors, img.id)
query = (Image