Add a chunk cleanup queue for async GC of empty chunks

Instead of having the Swift storage engine try to delete the empty chunk(s) synchronously, we simply queue them and have a worker come along after 30s to delete the empty chunks. This has a few key benefits: it is async (doesn't slow down the push code), helps deal with Swift's eventual consistency (less retries necessary) and is generic for other storage engines if/when they need this as well

test/test_swift.py

@@ -2,10 +2,11 @@ import io
 import unittest
 
 from collections import defaultdict
-
-from storage.swift import SwiftStorage
 from mock import MagicMock
 
+from storage import StorageContext
+from storage.swift import SwiftStorage
+
 
 class MockSwiftStorage(SwiftStorage):
   def __init__(self, *args, **kwargs):
@@ -18,7 +19,7 @@ class MockSwiftStorage(SwiftStorage):
 
 class MockSwiftTests(unittest.TestCase):
   base_args = {
-    'metric_queue': None,
+    'context': StorageContext('nyc', None, None),
     'swift_container': 'container-name',
     'storage_path': '/basepath',
     'auth_url': 'https://auth.com',
@@ -93,9 +94,26 @@ class FakeSwift(object):
     self.containers[container].pop(path, None)
 
 
+class FakeQueue(object):
+  def __init__(self):
+    self.items = []
+
+  def get(self):
+    if not self.items:
+      return None
+
+    return self.items.pop()
+
+  def put(self, names, item, available_after=0):
+    self.items.append({
+      'names': names,
+      'item': item,
+      'available_after': available_after,
+    })
+
 class FakeSwiftTests(unittest.TestCase):
   base_args = {
-    'metric_queue': None,
+    'context': StorageContext('nyc', None, None),
     'swift_container': 'container-name',
     'storage_path': '/basepath',
     'auth_url': 'https://auth.com',
@@ -170,6 +188,36 @@ class FakeSwiftTests(unittest.TestCase):
     for segment in SwiftStorage._segment_list_from_metadata(metadata):
       self.assertFalse(swift.exists(segment.path))
 
+  def test_empty_chunks_queued_for_deletion(self):
+    chunk_cleanup_queue = FakeQueue()
+    args = dict(self.base_args)
+    args['context'] = StorageContext('nyc', None, chunk_cleanup_queue)
+
+    swift = FakeSwiftStorage(**args)
+    uuid, metadata = swift.initiate_chunked_upload()
+
+    chunks = ['this', '', 'is', 'some', '', 'chunked', 'data', '']
+    offset = 0
+    for chunk in chunks:
+      length = len(chunk)
+      if length == 0:
+        length = 1
+
+      bytes_written, metadata, error = swift.stream_upload_chunk(uuid, offset, length,
+                                                                 io.BytesIO(chunk), metadata)
+      self.assertIsNone(error)
+      self.assertEquals(bytes_written, len(chunk))
+      offset += len(chunk)
+
+    swift.complete_chunked_upload(uuid, 'somepath', metadata)
+    self.assertEquals(''.join(chunks), swift.get_content('somepath'))
+
+    # Check the chunk deletion queue and ensure we have the last chunk queued.
+    found = chunk_cleanup_queue.get()
+    self.assertIsNotNone(found)
+
+    found2 = chunk_cleanup_queue.get()
+    self.assertIsNone(found2)
 
 if __name__ == '__main__':
   unittest.main()