From 29b8905051220e0e50ddccff35131866e77cbdc0 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 13 Oct 2016 11:44:50 -0400 Subject: [PATCH] Add retry and ignore around Swift deleting empty chunks Fixes #1979 --- storage/swift.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/storage/swift.py b/storage/swift.py index eded1b25f..355c04034 100644 --- a/storage/swift.py +++ b/storage/swift.py @@ -14,7 +14,7 @@ from swiftclient.client import Connection, ClientException from urlparse import urlparse from random import SystemRandom from hashlib import sha1 -from time import time +from time import time, sleep from collections import namedtuple from util.registry import filelike @@ -293,8 +293,18 @@ class SwiftStorage(BaseStorage): updated_metadata[_SEGMENTS_KEY].append(_PartUploadMetadata(segment_path, offset, bytes_written)) else: - # Delete the empty segment. - self.remove(segment_path) + # Try to delete the empty segment, as it is not needed. This will occasionally fail + # due to Swift's eventual consistency, so we retry a few times and then just leave it be. + for remaining_retries in range(2, -1, -1): + try: + self.remove(segment_path) + except IOError: + if remaining_retries: + sleep(0.25) + continue + + # Otherwise, ignore it. + break return bytes_written, updated_metadata