vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add S3 retry to all possible operations around the multipart upload

Browse source 
Fixes #1933
This commit is contained in:
Joseph Schorr 2016-10-04 20:18:33 +03:00
parent 94a0fee63f
commit 38415065e6
1 changed files with 10 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
storage/cloud.py
View file

@ -219,7 +219,7 @@ class _CloudStorage(BaseStorageV2):
self._metric_queue.put_deprecated('MultipartUploadSuccess', 1)
self._metric_queue.multipart_upload_end.Inc(labelvalues=['success'])
self._complete_s3_multipart_upload(mp, path)
self._perform_action_with_retry(mp.complete_upload)
return total_bytes_written, write_error
@ -348,12 +348,12 @@ class _CloudStorage(BaseStorageV2):
logger.exception(msg, chunk.path, final_path)
@staticmethod
def _complete_s3_multipart_upload(mpu, path):
def _perform_action_with_retry(action, *args, **kwargs):
# Note: Sometimes Amazon S3 simply raises an internal error when trying to complete a
# multipart upload. The recommendation is to simply try calling complete_upload again.
for remaining_retries in range(3, -1, -1):
# an action. The recommendation is to simply try calling the action again.
for remaining_retries in range(2, -1, -1):
try:
mpu.complete_upload()
action(*args, **kwargs)
break
except S3ResponseError as s3re:
if remaining_retries and s3re.status == 200 and s3re.error_code == 'InternalError':
@ -361,7 +361,7 @@ class _CloudStorage(BaseStorageV2):
continue
# Otherwise, raise it.
logger.exception('Exception trying to complete multipart upload for: %s', path)
logger.exception('Exception trying to perform action %s', action)
raise s3re
def complete_chunked_upload(self, uuid, final_path, storage_metadata, force_client_side=False):
@ -390,10 +390,11 @@ class _CloudStorage(BaseStorageV2):
abs_chunk_path = self._init_path(chunk.path)
part_num = chunk_offset + 1
chunk_end_offset_inclusive = chunk.length - 1
mpu.copy_part_from_key(self.get_cloud_bucket().name, abs_chunk_path, part_num,
start=0, end=chunk_end_offset_inclusive)
self._perform_action_with_retry(mpu.copy_part_from_key, self.get_cloud_bucket().name,
abs_chunk_path, part_num, start=0,
end=chunk_end_offset_inclusive)
self._complete_s3_multipart_upload(mpu, final_path)
self._perform_action_with_retry(mpu.complete_upload)
except IOError as ioe:
# Something bad happened, log it and then give up
msg = 'Exception when attempting server-side assembly for: %s'