From 0bc90ea45ba471150880352c6cebf46f04240b87 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 18 Aug 2016 11:56:23 -0400 Subject: [PATCH] Add retry attempts for internal error on multipart upload Fixes #1740 --- storage/cloud.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/storage/cloud.py b/storage/cloud.py index a9049ea58..e4929f5b6 100644 --- a/storage/cloud.py +++ b/storage/cloud.py @@ -3,6 +3,7 @@ import os import logging import copy +from boto.exception import S3ResponseError import boto.s3.connection import boto.s3.multipart import boto.gs.connection @@ -374,7 +375,22 @@ class _CloudStorage(BaseStorageV2): chunk_end_offset_inclusive = chunk.length - 1 mpu.copy_part_from_key(self.get_cloud_bucket().name, abs_chunk_path, part_num, start=0, end=chunk_end_offset_inclusive) - mpu.complete_upload() + + # Note: Sometimes Amazon S3 simply raises an internal error when trying to complete a + # multipart upload. The recommendation is to simply try calling complete_upload again. + for remaining_retries in range(3, -1, -1): + try: + mpu.complete_upload() + break + except S3ResponseError as s3re: + if remaining_retries and s3re.status == 200 and s3re.error_code == 'InternalError': + # Weird internal error case. Retry. + continue + + # Otherwise, raise it. + logger.exception('Exception trying to complete multipart upload for: %s', final_path) + raise s3re + except IOError as ioe: # Something bad happened, log it and then give up