vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Handle IOErrors in v2 uploads

Browse source
This commit is contained in:
Silas Sewell 2015-12-09 23:16:33 -05:00
parent 35437c9f55
commit 2dcc1f13a6
6 changed files with 63 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

17
storage/swift.py
View file

@ -269,15 +269,22 @@ class SwiftStorage(BaseStorage):
return random_uuid, metadata
def stream_upload_chunk(self, uuid, offset, length, in_fp, storage_metadata, content_type=None):
error = None
if length == 0:
return 0, storage_metadata
return 0, storage_metadata, error
# Note: Swift limits segments to a maximum of 5GB, so we keep writing segments until we
# are finished hitting the data limit.
total_bytes_written = 0
while True:
bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp,
storage_metadata, content_type)
try:
bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp,
storage_metadata, content_type)
except IOError as ex:
logger.warn('stream write error: %s', ex)
error = ex
break
if length != filelike.READ_UNTIL_END:
length = length - bytes_written
@ -285,7 +292,9 @@ class SwiftStorage(BaseStorage):
offset = offset + bytes_written
total_bytes_written = total_bytes_written + bytes_written
if bytes_written == 0 or length <= 0:
return total_bytes_written, storage_metadata
return total_bytes_written, storage_metadata, error
return total_bytes_written, storage_metadata, error
def _stream_upload_segment(self, uuid, offset, length, in_fp, storage_metadata, content_type):
updated_metadata = copy.deepcopy(storage_metadata)