vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add better logging to blob uploads

Browse source 
Fixes #1635
This commit is contained in:
Joseph Schorr 2016-07-20 17:53:43 -04:00
parent 4d6f96cd6c
commit cbf7c2bf44
4 changed files with 58 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

28
storage/swift.py
View file

@ -91,8 +91,8 @@ class SwiftStorage(BaseStorage):
_, obj = self._get_connection().get_object(self._swift_container, path,
resp_chunk_size=chunk_size)
return obj
except Exception:
logger.exception('Could not get object: %s', path)
except Exception as ex:
logger.exception('Could not get object at path %s: %s', path, ex)
raise IOError('Path %s not found' % path)
def _put_object(self, path, content, chunk=None, content_type=None, content_encoding=None,
@ -111,16 +111,16 @@ class SwiftStorage(BaseStorage):
# We re-raise client exception here so that validation of config during setup can see
# the client exception messages.
raise
except Exception:
logger.exception('Could not put object: %s', path)
except Exception as ex:
logger.exception('Could not put object at path %s: %s', path, ex)
raise IOError("Could not put content: %s" % path)
def _head_object(self, path):
path = self._normalize_path(path)
try:
return self._get_connection().head_object(self._swift_container, path)
except Exception:
logger.exception('Could not head object: %s', path)
except Exception as ex:
logger.exception('Could not head object at path %s: %s', path, ex)
return None
def get_direct_download_url(self, object_path, expires_in=60, requires_cors=False, head=False):
@ -233,22 +233,24 @@ class SwiftStorage(BaseStorage):
return random_uuid, metadata
def stream_upload_chunk(self, uuid, offset, length, in_fp, storage_metadata, content_type=None):
error = None
if length == 0:
return 0, storage_metadata, error
return 0, storage_metadata, None
# Note: Swift limits segments to a maximum of 5GB, so we keep writing segments until we
# are finished hitting the data limit.
total_bytes_written = 0
upload_error = None
while True:
try:
bytes_written, storage_metadata = self._stream_upload_segment(uuid, offset, length, in_fp,
storage_metadata,
content_type)
except IOError as ex:
logger.warn('stream write error: %s', ex)
error = ex
message = ('Error writing to stream in stream_upload_chunk for uuid %s (offset %s' +
', length %s, metadata: %s): %s')
logger.exception(message, uuid, offset, length, storage_metadata, ex)
upload_error = ex
break
if length != filelike.READ_UNTIL_END:
@ -257,9 +259,9 @@ class SwiftStorage(BaseStorage):
offset = offset + bytes_written
total_bytes_written = total_bytes_written + bytes_written
if bytes_written == 0 or length <= 0:
return total_bytes_written, storage_metadata, error
return total_bytes_written, storage_metadata, upload_error
return total_bytes_written, storage_metadata, error
return total_bytes_written, storage_metadata, upload_error
def _stream_upload_segment(self, uuid, offset, length, in_fp, storage_metadata, content_type):
updated_metadata = copy.deepcopy(storage_metadata)