vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add better logging to blob uploads

Browse source 
Fixes #1635
This commit is contained in:
Joseph Schorr 2016-07-20 17:53:43 -04:00
parent 4d6f96cd6c
commit cbf7c2bf44
4 changed files with 58 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

19
storage/cloud.py
View file

@ -174,7 +174,7 @@ class _CloudStorage(BaseStorageV2):
def _stream_write_internal(self, path, fp, content_type=None, content_encoding=None,
cancel_on_error=True, size=filelike.READ_UNTIL_END):
error = None
write_error = None
mp = self.__initiate_multipart_upload(path, content_type, content_encoding)
# We are going to reuse this but be VERY careful to only read the number of bytes written to it
@ -199,9 +199,9 @@ class _CloudStorage(BaseStorageV2):
mp.upload_part_from_file(buf, num_part, size=bytes_staged)
total_bytes_written += bytes_staged
num_part += 1
except IOError as ex:
logger.warn('stream write error: %s', ex)
error = ex
except IOError as e:
logger.warn('Error when writing to stream in stream_write_internal at path %s: %s', path, e)
write_error = e
if self._metric_queue is not None:
self._metric_queue.put_deprecated('MultipartUploadFailure', 1)
@ -209,7 +209,7 @@ class _CloudStorage(BaseStorageV2):
if cancel_on_error:
mp.cancel_upload()
return 0, error
return 0, write_error
else:
break
@ -220,7 +220,7 @@ class _CloudStorage(BaseStorageV2):
mp.complete_upload()
return total_bytes_written, error
return total_bytes_written, write_error
def exists(self, path):
self._initialize_cloud_conn()
@ -295,8 +295,9 @@ class _CloudStorage(BaseStorageV2):
# We are going to upload each chunk to a separate key
chunk_path = self._rel_upload_path(str(uuid4()))
bytes_written, error = self._stream_write_internal(chunk_path, in_fp, cancel_on_error=False,
size=length, content_type=content_type)
bytes_written, write_error = self._stream_write_internal(chunk_path, in_fp,
cancel_on_error=False, size=length,
content_type=content_type)
new_metadata = copy.deepcopy(storage_metadata)
@ -304,7 +305,7 @@ class _CloudStorage(BaseStorageV2):
if bytes_written > 0:
new_metadata[_CHUNKS_KEY].append(_PartUploadMetadata(chunk_path, offset, bytes_written))
return bytes_written, new_metadata, error
return bytes_written, new_metadata, write_error
def _chunk_generator(self, chunk_list):
for chunk in chunk_list: