Merge pull request #1511 from coreos-inc/location-cache

Use a cache for ImageStorageLocation

data/model/blob.py

@@ -2,8 +2,8 @@ from uuid import uuid4
 
 from data.model import (tag, _basequery, BlobDoesNotExist, InvalidBlobUpload, db_transaction,
                         storage as storage_model, InvalidImageException)
-from data.database import (Repository, Namespace, ImageStorage, Image, ImageStorageLocation,
-                           ImageStoragePlacement, BlobUpload)
+from data.database import (Repository, Namespace, ImageStorage, Image, ImageStoragePlacement,
+                           BlobUpload)
 
 
 def get_repo_blob_by_digest(namespace, repo_name, blob_digest):
@@ -75,6 +75,6 @@ def get_blob_upload(namespace, repo_name, upload_uuid):
 
 def initiate_upload(namespace, repo_name, uuid, location_name, storage_metadata):
   repo = _basequery.get_existing_repository(namespace, repo_name)
-  location = ImageStorageLocation.get(name=location_name)
-  return BlobUpload.create(repository=repo, location=location, uuid=uuid,
+  location = storage_model.get_image_location_for_name(location_name)
+  return BlobUpload.create(repository=repo, location=location.id, uuid=uuid,
                            storage_metadata=storage_metadata)

data/model/storage.py

@@ -1,6 +1,8 @@
 import logging
 
 from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
+from cachetools import lru_cache
+from collections import namedtuple
 
 from data.model import (config, db_transaction, InvalidImageException, TorrentInfoDoesNotExist,
                         DataModelException, _basequery)
@@ -11,11 +13,33 @@ from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStor
 
 logger = logging.getLogger(__name__)
 
+_Location = namedtuple('location', ['id', 'name'])
+
+@lru_cache(maxsize=1)
+def get_image_locations():
+  location_map = {}
+  for location in ImageStorageLocation.select():
+    location_tuple = _Location(location.id, location.name)
+    location_map[location.id] = location_tuple
+    location_map[location.name] = location_tuple
+
+  return location_map
+
+
+def get_image_location_for_name(location_name):
+  locations = get_image_locations()
+  return locations[location_name]
+
+
+def get_image_location_for_id(location_id):
+  locations = get_image_locations()
+  return locations[location_id]
+
 
 def add_storage_placement(storage, location_name):
   """ Adds a storage placement for the given storage at the given location. """
-  location = ImageStorageLocation.get(name=location_name)
-  ImageStoragePlacement.create(location=location, storage=storage)
+  location = get_image_location_for_name(location_name)
+  ImageStoragePlacement.create(location=location.id, storage=storage)
 
 
 def garbage_collect_storage(storage_id_whitelist):
@@ -23,7 +47,8 @@ def garbage_collect_storage(storage_id_whitelist):
     return
 
   def placements_query_to_paths_set(placements_query):
-    return {(placement.location.name, get_layer_path(placement.storage))
+    return {(get_image_location_for_id(placement.location_id).name,
+             get_layer_path(placement.storage))
             for placement in placements_query}
 
   def orphaned_storage_query(select_base_query, candidates, group_by):
@@ -43,14 +68,10 @@ def garbage_collect_storage(storage_id_whitelist):
     # Track all of the data that should be removed from blob storage
     placements_to_remove = list(orphaned_storage_query(ImageStoragePlacement
                                                        .select(ImageStoragePlacement,
-                                                               ImageStorage,
-                                                               ImageStorageLocation)
-                                                       .join(ImageStorageLocation)
-                                                       .switch(ImageStoragePlacement)
+                                                               ImageStorage)
                                                        .join(ImageStorage),
                                                        storage_id_whitelist,
-                                                       (ImageStorage.id, ImageStoragePlacement.id,
-                                                        ImageStorageLocation.id)))
+                                                       (ImageStorage.id, ImageStoragePlacement.id)))
 
     paths_to_remove = placements_query_to_paths_set(placements_to_remove)
 
@@ -97,8 +118,8 @@ def garbage_collect_storage(storage_id_whitelist):
 
 def create_v1_storage(location_name):
   storage = ImageStorage.create(cas_path=False)
-  location = ImageStorageLocation.get(name=location_name)
-  ImageStoragePlacement.create(location=location, storage=storage)
+  location = get_image_location_for_name(location_name)
+  ImageStoragePlacement.create(location=location.id, storage=storage)
   storage.locations = {location_name}
   return storage
 
@@ -125,8 +146,7 @@ def lookup_storage_signature(storage, signature_kind_name):
 
 def _get_storage(query_modifier):
   query = (ImageStoragePlacement
-           .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
-           .join(ImageStorageLocation)
+           .select(ImageStoragePlacement, ImageStorage)
            .switch(ImageStoragePlacement)
            .join(ImageStorage))
 
@@ -136,8 +156,8 @@ def _get_storage(query_modifier):
     raise InvalidImageException()
 
   found = placements[0].storage
-  found.locations = {placement.location.name for placement in placements}
-
+  found.locations = {get_image_location_for_id(placement.location_id).name
+                     for placement in placements}
   return found
 
 
@@ -253,12 +273,10 @@ def set_image_storage_metadata(docker_image_id, namespace_name, repository_name,
 def get_storage_locations(uuid):
   query = (ImageStoragePlacement
            .select()
-           .join(ImageStorageLocation)
-           .switch(ImageStoragePlacement)
            .join(ImageStorage)
            .where(ImageStorage.uuid == uuid))
 
-  return [location.location.name for location in query]
+  return [get_image_location_for_id(placement.location_id).name for placement in query]
 
 
 def save_torrent_info(storage_object, piece_length, pieces):