Add retry and ignore around Swift deleting empty chunks

Fixes #1979

storage/swift.py

@@ -14,7 +14,7 @@ from swiftclient.client import Connection, ClientException
 from urlparse import urlparse
 from random import SystemRandom
 from hashlib import sha1
-from time import time
+from time import time, sleep
 from collections import namedtuple
 
 from util.registry import filelike
@@ -293,8 +293,18 @@ class SwiftStorage(BaseStorage):
       updated_metadata[_SEGMENTS_KEY].append(_PartUploadMetadata(segment_path, offset,
                                                                  bytes_written))
     else:
-      # Delete the empty segment.
-      self.remove(segment_path)
+      # Try to delete the empty segment, as it is not needed. This will occasionally fail
+      # due to Swift's eventual consistency, so we retry a few times and then just leave it be.
+      for remaining_retries in range(2, -1, -1):
+        try:
+          self.remove(segment_path)
+        except IOError:
+          if remaining_retries:
+            sleep(0.25)
+            continue
+
+          # Otherwise, ignore it.
+          break
 
     return bytes_written, updated_metadata