Improve the performance of fetching manifest blobs by checksum.
This commit is contained in:
parent
790a33589d
commit
b564492ea7
2 changed files with 23 additions and 10 deletions
|
@ -1,6 +1,6 @@
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from peewee import JOIN_LEFT_OUTER, fn
|
from peewee import JOIN_LEFT_OUTER, fn, SQL
|
||||||
|
|
||||||
from data.model import config, db_transaction, InvalidImageException
|
from data.model import config, db_transaction, InvalidImageException
|
||||||
from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStoragePlacement,
|
from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStoragePlacement,
|
||||||
|
@ -218,13 +218,26 @@ def get_layer_path(storage_record):
|
||||||
|
|
||||||
return store.blob_path(storage_record.content_checksum)
|
return store.blob_path(storage_record.content_checksum)
|
||||||
|
|
||||||
|
|
||||||
def lookup_repo_storages_by_content_checksum(repo, checksums):
|
def lookup_repo_storages_by_content_checksum(repo, checksums):
|
||||||
""" Looks up repository storages (without placements) matching the given repository
|
""" Looks up repository storages (without placements) matching the given repository
|
||||||
and checksum. """
|
and checksum. """
|
||||||
return (ImageStorage
|
# There may be many duplicates of the checksums, so for performance reasons we are going
|
||||||
.select()
|
# to use a union to select just one storage with each checksum
|
||||||
|
queries = []
|
||||||
|
|
||||||
|
for checksum in set(checksums):
|
||||||
|
candidate_subq = (ImageStorage
|
||||||
|
.select(ImageStorage.id, ImageStorage.content_checksum)
|
||||||
.join(Image)
|
.join(Image)
|
||||||
.where(Image.repository == repo, ImageStorage.content_checksum << checksums))
|
.where(Image.repository == repo, ImageStorage.content_checksum == checksum)
|
||||||
|
.limit(1))
|
||||||
|
queries.append(ImageStorage
|
||||||
|
.select(SQL('*'))
|
||||||
|
.from_(candidate_subq))
|
||||||
|
|
||||||
|
return reduce(lambda l, r: l.union_all(r), queries)
|
||||||
|
|
||||||
|
|
||||||
def get_storage_locations(uuid):
|
def get_storage_locations(uuid):
|
||||||
query = (ImageStoragePlacement
|
query = (ImageStoragePlacement
|
||||||
|
|
|
@ -287,16 +287,16 @@ def _write_manifest(namespace, repo_name, manifest):
|
||||||
# know which V1 images we need to synthesize and which ones are invalid.
|
# know which V1 images we need to synthesize and which ones are invalid.
|
||||||
layers = list(manifest.layers)
|
layers = list(manifest.layers)
|
||||||
|
|
||||||
docker_image_ids = [mdata.v1_metadata.docker_id for mdata in layers]
|
docker_image_ids = {mdata.v1_metadata.docker_id for mdata in layers}
|
||||||
parent_image_ids = [mdata.v1_metadata.parent for mdata in layers
|
parent_image_ids = {mdata.v1_metadata.parent for mdata in layers
|
||||||
if mdata.v1_metadata.parent]
|
if mdata.v1_metadata.parent}
|
||||||
all_image_ids = list(set(docker_image_ids + parent_image_ids))
|
all_image_ids = list(docker_image_ids | parent_image_ids)
|
||||||
|
|
||||||
images_query = model.image.lookup_repository_images(repo, all_image_ids)
|
images_query = model.image.lookup_repository_images(repo, all_image_ids)
|
||||||
images_map = {image.docker_image_id: image for image in images_query}
|
images_map = {image.docker_image_id: image for image in images_query}
|
||||||
|
|
||||||
# Lookup the storages associated with each blob in the manifest.
|
# Lookup the storages associated with each blob in the manifest.
|
||||||
checksums = [str(mdata.digest) for mdata in manifest.layers]
|
checksums = list({str(mdata.digest) for mdata in manifest.layers})
|
||||||
storage_query = model.storage.lookup_repo_storages_by_content_checksum(repo, checksums)
|
storage_query = model.storage.lookup_repo_storages_by_content_checksum(repo, checksums)
|
||||||
storage_map = {storage.content_checksum: storage for storage in storage_query}
|
storage_map = {storage.content_checksum: storage for storage in storage_query}
|
||||||
|
|
||||||
|
|
Reference in a new issue