Merge remote-tracking branch 'origin/laffa' into nomenclature

Conflicts:
	test/data/test.db

data/model/legacy.py

@@ -13,7 +13,7 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor
                            Notification, ImageStorageLocation, ImageStoragePlacement,
                            ExternalNotificationEvent, ExternalNotificationMethod,
                            RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
-                           random_string_generator, db, BUILD_PHASE)
+                           DerivedImageStorage, random_string_generator, db, BUILD_PHASE)
 from peewee import JOIN_LEFT_OUTER, fn
 from util.validation import (validate_username, validate_email, validate_password,
                              INVALID_PASSWORD_MESSAGE)
@@ -1161,6 +1161,14 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe
   return '/%s/' % '/'.join(new_ids)
 
 
+def _create_storage(location_name):
+  storage = ImageStorage.create()
+  location = ImageStorageLocation.get(name=location_name)
+  ImageStoragePlacement.create(location=location, storage=storage)
+  storage.locations = {location_name}
+  return storage
+
+
 def find_create_or_link_image(docker_image_id, repository, username, translations,
                               preferred_location):
   with config.app_config['DB_TRANSACTION_FACTORY'](db):
@@ -1201,10 +1209,7 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
       origin_image_id = to_copy.id
     except Image.DoesNotExist:
       logger.debug('Creating new storage for docker id: %s', docker_image_id)
-      storage = ImageStorage.create()
-      location = ImageStorageLocation.get(name=preferred_location)
-      ImageStoragePlacement.create(location=location, storage=storage)
-      storage.locations = {preferred_location}
+      storage = _create_storage(preferred_location)
 
     logger.debug('Storage locations: %s', storage.locations)
 
@@ -1222,6 +1227,26 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
     return new_image
 
 
+def find_or_create_derived_storage(source, transformation_name, preferred_location):
+  try:
+    found = (ImageStorage
+      .select(ImageStorage, DerivedImageStorage)
+      .join(DerivedImageStorage, on=(ImageStorage.id == DerivedImageStorage.derivative))
+      .join(ImageStorageTransformation)
+      .where(DerivedImageStorage.source == source,
+             ImageStorageTransformation.name == transformation_name)
+      .get())
+
+    found.locations = {placement.location.name for placement in found.imagestorageplacement_set}
+    return found
+  except ImageStorage.DoesNotExist:
+    logger.debug('Creating storage dervied from source: %s', source.uuid)
+    trans = ImageStorageTransformation.get(name=transformation_name)
+    new_storage = _create_storage(preferred_location)
+    DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
+    return new_storage
+
+
 def get_storage_by_uuid(storage_uuid):
   placements = list(ImageStoragePlacement
                     .select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
@@ -1377,15 +1402,8 @@ def garbage_collect_repository(namespace_name, repository_name):
 
       image_to_remove.delete_instance()
 
-    if uuids_to_check_for_gc:
-      storage_to_remove = (ImageStorage
-                           .select()
-                           .join(Image, JOIN_LEFT_OUTER)
-                           .group_by(ImageStorage)
-                           .where(ImageStorage.uuid << list(uuids_to_check_for_gc))
-                           .having(fn.Count(Image.id) == 0))
-
-      for storage in storage_to_remove:
+    def remove_storages(query):
+      for storage in query:
         logger.debug('Garbage collecting image storage: %s', storage.uuid)
 
         image_path = config.store.image_path(storage.uuid)      
@@ -1394,7 +1412,24 @@ def garbage_collect_repository(namespace_name, repository_name):
           placement.delete_instance()
           config.store.remove({location_name}, image_path)
 
-        storage.delete_instance()
+        storage.delete_instance(recursive=True)
+
+    if uuids_to_check_for_gc:
+      storage_to_remove = (ImageStorage
+                           .select()
+                           .join(Image, JOIN_LEFT_OUTER)
+                           .group_by(ImageStorage)
+                           .where(ImageStorage.uuid << list(uuids_to_check_for_gc))
+                           .having(fn.Count(Image.id) == 0))
+
+      remove_storages(storage_to_remove)
+
+    # Now remove any derived image storages whose sources have been removed
+    derived_storages_to_remove = (ImageStorage
+      .select()
+      .join(DerivedImageStorage, on=(ImageStorage.id == DerivedImageStorage.derivative))
+      .where(DerivedImageStorage.source >> None))
+    remove_storages(derived_storages_to_remove)
 
     return len(to_remove)