vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add mocked unit tests for cloud storage engine

Browse source
This commit is contained in:
Joseph Schorr 2016-03-22 18:16:48 -04:00
parent d33804c22e
commit eab6af2b87
4 changed files with 168 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

31
storage/cloud.py
View file

@ -163,7 +163,9 @@ class _CloudStorage(BaseStorageV2):
if content_encoding is not None:
metadata['Content-Encoding'] = content_encoding
self._metric_queue.put('MultipartUploadStart', 1)
if self._metric_queue is not None:
self._metric_queue.put('MultipartUploadStart', 1)
return self._cloud_bucket.initiate_multipart_upload(path, metadata=metadata,
**self._upload_params)
@ -200,7 +202,10 @@ class _CloudStorage(BaseStorageV2):
except IOError as ex:
logger.warn('stream write error: %s', ex)
error = ex
self._metric_queue.put('MultipartUploadFailure', 1)
if self._metric_queue is not None:
self._metric_queue.put('MultipartUploadFailure', 1)
if cancel_on_error:
mp.cancel_upload()
return 0, error
@ -208,7 +213,9 @@ class _CloudStorage(BaseStorageV2):
break
if total_bytes_written > 0:
self._metric_queue.put('MultipartUploadSuccess', 1)
if self._metric_queue is not None:
self._metric_queue.put('MultipartUploadSuccess', 1)
mp.complete_upload()
return total_bytes_written, error
@ -334,18 +341,20 @@ class _CloudStorage(BaseStorageV2):
msg = 'Failed to clean up chunk %s for reupload of %s'
logger.exception(msg, chunk.path, final_path)
def complete_chunked_upload(self, uuid, final_path, storage_metadata):
def complete_chunked_upload(self, uuid, final_path, storage_metadata, force_client_side=False):
self._initialize_cloud_conn()
chunk_list = self._chunk_list_from_metadata(storage_metadata)
# Here is where things get interesting: we are going to try to assemble this server side
# In order to be a candidate all parts (after offsets have been computed) must be at least 5MB
server_side_assembly = True
chunk_list = self._chunk_list_from_metadata(storage_metadata)
for chunk_offset, chunk in enumerate(chunk_list):
# If the chunk is both too small, and not the last chunk, we rule out server side assembly
if chunk.length < self.automatic_chunk_size and (chunk_offset + 1) < len(chunk_list):
server_side_assembly = False
break
server_side_assembly = False
if not force_client_side:
server_side_assembly = True
for chunk_offset, chunk in enumerate(chunk_list):
# If the chunk is both too small, and not the last chunk, we rule out server side assembly
if chunk.length < self.automatic_chunk_size and (chunk_offset + 1) < len(chunk_list):
server_side_assembly = False
break
if server_side_assembly:
logger.debug('Performing server side assembly of multi-part upload for: %s', final_path)