From 7aa6b812e264e8a5336f0f2dbe271ae457a401a0 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 2 Jun 2016 15:57:59 -0400 Subject: [PATCH] Use a cache for ImageStorageLocation No need to reload it from the DB or join as it is a static set only changed during migration --- data/model/blob.py | 8 +++---- data/model/storage.py | 54 ++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/data/model/blob.py b/data/model/blob.py index c05fac910..2584d96f2 100644 --- a/data/model/blob.py +++ b/data/model/blob.py @@ -2,8 +2,8 @@ from uuid import uuid4 from data.model import (tag, _basequery, BlobDoesNotExist, InvalidBlobUpload, db_transaction, storage as storage_model, InvalidImageException) -from data.database import (Repository, Namespace, ImageStorage, Image, ImageStorageLocation, - ImageStoragePlacement, BlobUpload) +from data.database import (Repository, Namespace, ImageStorage, Image, ImageStoragePlacement, + BlobUpload) def get_repo_blob_by_digest(namespace, repo_name, blob_digest): @@ -75,6 +75,6 @@ def get_blob_upload(namespace, repo_name, upload_uuid): def initiate_upload(namespace, repo_name, uuid, location_name, storage_metadata): repo = _basequery.get_existing_repository(namespace, repo_name) - location = ImageStorageLocation.get(name=location_name) - return BlobUpload.create(repository=repo, location=location, uuid=uuid, + location = storage_model.get_image_location_for_name(location_name) + return BlobUpload.create(repository=repo, location=location.id, uuid=uuid, storage_metadata=storage_metadata) diff --git a/data/model/storage.py b/data/model/storage.py index 0f2220f91..6f7afde09 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -1,6 +1,8 @@ import logging from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError +from cachetools import lru_cache +from collections import namedtuple from data.model import (config, db_transaction, InvalidImageException, TorrentInfoDoesNotExist, DataModelException, _basequery) @@ -11,11 +13,33 @@ from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStor logger = logging.getLogger(__name__) +_Location = namedtuple('location', ['id', 'name']) + +@lru_cache(maxsize=1) +def get_image_locations(): + location_map = {} + for location in ImageStorageLocation.select(): + location_tuple = _Location(location.id, location.name) + location_map[location.id] = location_tuple + location_map[location.name] = location_tuple + + return location_map + + +def get_image_location_for_name(location_name): + locations = get_image_locations() + return locations[location_name] + + +def get_image_location_for_id(location_id): + locations = get_image_locations() + return locations[location_id] + def add_storage_placement(storage, location_name): """ Adds a storage placement for the given storage at the given location. """ - location = ImageStorageLocation.get(name=location_name) - ImageStoragePlacement.create(location=location, storage=storage) + location = get_image_location_for_name(location_name) + ImageStoragePlacement.create(location=location.id, storage=storage) def garbage_collect_storage(storage_id_whitelist): @@ -23,7 +47,8 @@ def garbage_collect_storage(storage_id_whitelist): return def placements_query_to_paths_set(placements_query): - return {(placement.location.name, get_layer_path(placement.storage)) + return {(get_image_location_for_id(placement.location_id).name, + get_layer_path(placement.storage)) for placement in placements_query} def orphaned_storage_query(select_base_query, candidates, group_by): @@ -43,14 +68,10 @@ def garbage_collect_storage(storage_id_whitelist): # Track all of the data that should be removed from blob storage placements_to_remove = list(orphaned_storage_query(ImageStoragePlacement .select(ImageStoragePlacement, - ImageStorage, - ImageStorageLocation) - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) + ImageStorage) .join(ImageStorage), storage_id_whitelist, - (ImageStorage.id, ImageStoragePlacement.id, - ImageStorageLocation.id))) + (ImageStorage.id, ImageStoragePlacement.id))) paths_to_remove = placements_query_to_paths_set(placements_to_remove) @@ -97,8 +118,8 @@ def garbage_collect_storage(storage_id_whitelist): def create_v1_storage(location_name): storage = ImageStorage.create(cas_path=False) - location = ImageStorageLocation.get(name=location_name) - ImageStoragePlacement.create(location=location, storage=storage) + location = get_image_location_for_name(location_name) + ImageStoragePlacement.create(location=location.id, storage=storage) storage.locations = {location_name} return storage @@ -125,8 +146,7 @@ def lookup_storage_signature(storage, signature_kind_name): def _get_storage(query_modifier): query = (ImageStoragePlacement - .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation) - .join(ImageStorageLocation) + .select(ImageStoragePlacement, ImageStorage) .switch(ImageStoragePlacement) .join(ImageStorage)) @@ -136,8 +156,8 @@ def _get_storage(query_modifier): raise InvalidImageException() found = placements[0].storage - found.locations = {placement.location.name for placement in placements} - + found.locations = {get_image_location_for_id(placement.location_id).name + for placement in placements} return found @@ -252,12 +272,10 @@ def set_image_storage_metadata(docker_image_id, namespace_name, repository_name, def get_storage_locations(uuid): query = (ImageStoragePlacement .select() - .join(ImageStorageLocation) - .switch(ImageStoragePlacement) .join(ImageStorage) .where(ImageStorage.uuid == uuid)) - return [location.location.name for location in query] + return [get_image_location_for_id(placement.location_id).name for placement in query] def save_torrent_info(storage_object, piece_length, pieces):