Actually store the generated image storage in the database, and allow it to be garbage collected when the parent image storage is collected.

data/model/legacy.py

@@ -70,6 +70,10 @@ class InvalidBuildTriggerException(DataModelException):
   pass
 
 
+class InvalidImageException(DataModelException):
+  pass
+
+
 class TooManyUsersException(DataModelException):
   pass
 
@@ -1055,6 +1059,14 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe
   return '/%s/' % '/'.join(new_ids)
 
 
+def _create_storage(location_name):
+  storage = ImageStorage.create()
+  location = ImageStorageLocation.get(name=location_name)
+  ImageStoragePlacement.create(location=location, storage=storage)
+  storage.locations = {location_name}
+  return storage
+
+
 def find_create_or_link_image(docker_image_id, repository, username, translations,
                               preferred_location):
   with config.app_config['DB_TRANSACTION_FACTORY'](db):
@@ -1093,10 +1105,7 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
       origin_image_id = to_copy.id
     except Image.DoesNotExist:
       logger.debug('Creating new storage for docker id: %s', docker_image_id)
-      storage = ImageStorage.create()
-      location = ImageStorageLocation.get(name=preferred_location)
-      ImageStoragePlacement.create(location=location, storage=storage)
-      storage.locations = {preferred_location}
+      storage = _create_storage(preferred_location)
 
     logger.debug('Storage locations: %s', storage.locations)
 
@@ -1114,6 +1123,43 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
     return new_image
 
 
+def find_or_create_derived_storage(source, transformation_name, preferred_location):
+  try:
+    found = (ImageStorage
+      .select(ImageStorage, DerivedImageStorage)
+      .join(DerivedImageStorage, on=(ImageStorage.id == DerivedImageStorage.derivative))
+      .join(ImageStorageTransformation)
+      .where(DerivedImageStorage.source == source,
+             ImageStorageTransformation.name == transformation_name)
+      .get())
+
+    found.locations = {placement.location.name for placement in found.imagestorageplacement_set}
+    return found
+  except ImageStorage.DoesNotExist:
+    logger.debug('Creating storage dervied from source: %s', source.uuid)
+    trans = ImageStorageTransformation.get(name=transformation_name)
+    new_storage = _create_storage(preferred_location)
+    DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
+    return new_storage
+
+
+def get_storage_by_uuid(storage_uuid):
+  placements = list(ImageStoragePlacement
+    .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
+    .join(ImageStorageLocation)
+    .switch(ImageStoragePlacement)
+    .join(ImageStorage)
+    .where(ImageStorage.uuid == storage_uuid))
+
+  if not placements:
+    raise InvalidImageException('No storage found with uuid: %s', storage_uuid)
+
+  found = placements[0].storage
+  found.locations = {placement.location.name for placement in placements}
+
+  return found
+
+
 def set_image_size(docker_image_id, namespace_name, repository_name,
                    image_size):
   try:
@@ -1252,15 +1298,8 @@ def garbage_collect_repository(namespace_name, repository_name):
 
       image_to_remove.delete_instance()
 
-    if uuids_to_check_for_gc:
-      storage_to_remove = (ImageStorage
-        .select()
-        .join(Image, JOIN_LEFT_OUTER)
-        .group_by(ImageStorage)
-        .where(ImageStorage.uuid << list(uuids_to_check_for_gc))
-        .having(fn.Count(Image.id) == 0))
-
-      for storage in storage_to_remove:
+    def remove_storages(query):
+      for storage in query:
         logger.debug('Garbage collecting image storage: %s', storage.uuid)
 
         image_path = config.store.image_path(storage.uuid)      
@@ -1269,7 +1308,24 @@ def garbage_collect_repository(namespace_name, repository_name):
           placement.delete_instance()
           config.store.remove({location_name}, image_path)
 
-        storage.delete_instance()
+        storage.delete_instance(recursive=True)
+
+    if uuids_to_check_for_gc:
+      storage_to_remove = (ImageStorage
+        .select()
+        .join(Image, JOIN_LEFT_OUTER)
+        .group_by(ImageStorage)
+        .where(ImageStorage.uuid << list(uuids_to_check_for_gc))
+        .having(fn.Count(Image.id) == 0))
+
+      remove_storages(storage_to_remove)
+
+    # Now remove any derived image storages whose sources have been removed
+    derived_storages_to_remove = (ImageStorage
+      .select()
+      .join(DerivedImageStorage, on=(ImageStorage.id == DerivedImageStorage.derivative))
+      .where(DerivedImageStorage.source >> None))
+    remove_storages(derived_storages_to_remove)
 
     return len(to_remove)