Add a chunk cleanup queue for async GC of empty chunks

Instead of having the Swift storage engine try to delete the empty chunk(s) synchronously, we simply queue them and have a worker come along after 30s to delete the empty chunks. This has a few key benefits: it is async (doesn't slow down the push code), helps deal with Swift's eventual consistency (less retries necessary) and is generic for other storage engines if/when they need this as well

workers/chunkcleanupworker.py

@@ -0,0 +1,39 @@
+import logging
+import time
+
+from app import app, storage, chunk_cleanup_queue
+from workers.queueworker import QueueWorker, JobException
+
+logger = logging.getLogger(__name__)
+
+
+POLL_PERIOD_SECONDS = 10
+
+
+class ChunkCleanupWorker(QueueWorker):
+  """ Worker which cleans up chunks enqueued by the storage engine(s). This is typically used to
+      cleanup empty chunks which are no longer needed.
+  """
+  def process_queue_item(self, job_details):
+    logger.debug('Got chunk cleanup queue item: %s', job_details)
+    storage_location = job_details['location']
+    storage_path = job_details['path']
+
+    try:
+      storage.remove([storage_location], storage_path)
+    except IOError:
+      raise JobException()
+
+
+if __name__ == "__main__":
+  logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
+
+  engines = set([config[0] for config in app.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values()])
+  if 'SwiftStorage' not in engines:
+    logger.debug('Swift storage not detected; sleeping')
+    while True:
+      time.sleep(10000)
+
+  logger.debug('Starting chunk cleanup worker')
+  worker = ChunkCleanupWorker(chunk_cleanup_queue, poll_period_seconds=POLL_PERIOD_SECONDS)
+  worker.start()