vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #1940 from coreos-inc/s3-retry

Browse source 
Add S3 retry to all possible operations around the multipart upload
This commit is contained in:
josephschorr 2016-10-05 23:02:22 +03:00 committed by GitHub
commit a43f617efe
1 changed files with 10 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
storage/cloud.py
View file

@ -219,7 +219,7 @@ class _CloudStorage(BaseStorageV2):
self._metric_queue.put_deprecated('MultipartUploadSuccess', 1) self._metric_queue.put_deprecated('MultipartUploadSuccess', 1)
self._metric_queue.multipart_upload_end.Inc(labelvalues=['success']) self._metric_queue.multipart_upload_end.Inc(labelvalues=['success'])
self._complete_s3_multipart_upload(mp, path) self._perform_action_with_retry(mp.complete_upload)
return total_bytes_written, write_error return total_bytes_written, write_error
@ -348,12 +348,12 @@ class _CloudStorage(BaseStorageV2):
logger.exception(msg, chunk.path, final_path) logger.exception(msg, chunk.path, final_path)
@staticmethod @staticmethod
def _complete_s3_multipart_upload(mpu, path): def _perform_action_with_retry(action, *args, **kwargs):
# Note: Sometimes Amazon S3 simply raises an internal error when trying to complete a # Note: Sometimes Amazon S3 simply raises an internal error when trying to complete a
# multipart upload. The recommendation is to simply try calling complete_upload again. # an action. The recommendation is to simply try calling the action again.
for remaining_retries in range(3, -1, -1): for remaining_retries in range(2, -1, -1):
try: try:
mpu.complete_upload() action(*args, **kwargs)
break break
except S3ResponseError as s3re: except S3ResponseError as s3re:
if remaining_retries and s3re.status == 200 and s3re.error_code == 'InternalError': if remaining_retries and s3re.status == 200 and s3re.error_code == 'InternalError':
@ -361,7 +361,7 @@ class _CloudStorage(BaseStorageV2):
continue continue
# Otherwise, raise it. # Otherwise, raise it.
logger.exception('Exception trying to complete multipart upload for: %s', path) logger.exception('Exception trying to perform action %s', action)
raise s3re raise s3re
def complete_chunked_upload(self, uuid, final_path, storage_metadata, force_client_side=False): def complete_chunked_upload(self, uuid, final_path, storage_metadata, force_client_side=False):
@ -390,10 +390,11 @@ class _CloudStorage(BaseStorageV2):
abs_chunk_path = self._init_path(chunk.path) abs_chunk_path = self._init_path(chunk.path)
part_num = chunk_offset + 1 part_num = chunk_offset + 1
chunk_end_offset_inclusive = chunk.length - 1 chunk_end_offset_inclusive = chunk.length - 1
mpu.copy_part_from_key(self.get_cloud_bucket().name, abs_chunk_path, part_num, self._perform_action_with_retry(mpu.copy_part_from_key, self.get_cloud_bucket().name,
start=0, end=chunk_end_offset_inclusive) abs_chunk_path, part_num, start=0,
end=chunk_end_offset_inclusive)
self._complete_s3_multipart_upload(mpu, final_path) self._perform_action_with_retry(mpu.complete_upload)
except IOError as ioe: except IOError as ioe:
# Something bad happened, log it and then give up # Something bad happened, log it and then give up
msg = 'Exception when attempting server-side assembly for: %s' msg = 'Exception when attempting server-side assembly for: %s'