diff --git a/storage/cloud.py b/storage/cloud.py index de4e22691..1777c1fb2 100644 --- a/storage/cloud.py +++ b/storage/cloud.py @@ -219,7 +219,7 @@ class _CloudStorage(BaseStorageV2): self._metric_queue.put_deprecated('MultipartUploadSuccess', 1) self._metric_queue.multipart_upload_end.Inc(labelvalues=['success']) - self._complete_s3_multipart_upload(mp, path) + self._perform_action_with_retry(mp.complete_upload) return total_bytes_written, write_error @@ -348,12 +348,12 @@ class _CloudStorage(BaseStorageV2): logger.exception(msg, chunk.path, final_path) @staticmethod - def _complete_s3_multipart_upload(mpu, path): + def _perform_action_with_retry(action, *args, **kwargs): # Note: Sometimes Amazon S3 simply raises an internal error when trying to complete a - # multipart upload. The recommendation is to simply try calling complete_upload again. - for remaining_retries in range(3, -1, -1): + # an action. The recommendation is to simply try calling the action again. + for remaining_retries in range(2, -1, -1): try: - mpu.complete_upload() + action(*args, **kwargs) break except S3ResponseError as s3re: if remaining_retries and s3re.status == 200 and s3re.error_code == 'InternalError': @@ -361,7 +361,7 @@ class _CloudStorage(BaseStorageV2): continue # Otherwise, raise it. - logger.exception('Exception trying to complete multipart upload for: %s', path) + logger.exception('Exception trying to perform action %s', action) raise s3re def complete_chunked_upload(self, uuid, final_path, storage_metadata, force_client_side=False): @@ -390,10 +390,11 @@ class _CloudStorage(BaseStorageV2): abs_chunk_path = self._init_path(chunk.path) part_num = chunk_offset + 1 chunk_end_offset_inclusive = chunk.length - 1 - mpu.copy_part_from_key(self.get_cloud_bucket().name, abs_chunk_path, part_num, - start=0, end=chunk_end_offset_inclusive) + self._perform_action_with_retry(mpu.copy_part_from_key, self.get_cloud_bucket().name, + abs_chunk_path, part_num, start=0, + end=chunk_end_offset_inclusive) - self._complete_s3_multipart_upload(mpu, final_path) + self._perform_action_with_retry(mpu.complete_upload) except IOError as ioe: # Something bad happened, log it and then give up msg = 'Exception when attempting server-side assembly for: %s'