vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

override upload_chunk_complete for RadosGW

Browse source 
RadosGW doesn't support server-side copy of keys into multipart, so we
have to always join it on the local side.
This commit is contained in:
Jimmy Zelinskie 2015-09-30 15:11:28 -04:00
parent a3ebb9028d
commit abe43a0e07
1 changed files with 19 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
storage/cloud.py
View file

@ -342,8 +342,8 @@ class _CloudStorage(BaseStorageV2):
abs_chunk_path = self._init_path(chunk.path) abs_chunk_path = self._init_path(chunk.path)
part_num = chunk_offset + 1 part_num = chunk_offset + 1
chunk_end_offset_inclusive = chunk.length - 1 chunk_end_offset_inclusive = chunk.length - 1
mpu.copy_part_from_key(self.get_cloud_bucket().name, abs_chunk_path, part_num, 0, mpu.copy_part_from_key(self.get_cloud_bucket().name, abs_chunk_path, part_num,
chunk_end_offset_inclusive) start=0, end=chunk_end_offset_inclusive)
mpu.complete_upload() mpu.complete_upload()
except IOError as ioe: except IOError as ioe:
@ -354,10 +354,9 @@ class _CloudStorage(BaseStorageV2):
raise ioe raise ioe
else: else:
logger.warning('Performing client side assmebly of multi-part upload for: %s', final_path)
# We are going to turn all of the server side objects into a single file-like stream, and # We are going to turn all of the server side objects into a single file-like stream, and
# pass that to stream_write to chunk and upload the final object. # pass that to stream_write to chunk and upload the final object.
logger.warning('Performing client side assmebly of multi-part upload for: %s', final_path)
concatenated = filelike.FilelikeStreamConcat(self._chunk_generator(chunk_list)) concatenated = filelike.FilelikeStreamConcat(self._chunk_generator(chunk_list))
self.stream_write(final_path, concatenated) self.stream_write(final_path, concatenated)
@ -472,3 +471,19 @@ class RadosGWStorage(_CloudStorage):
return None return None
return super(RadosGWStorage, self).get_direct_upload_url(path, mime_type, requires_cors) return super(RadosGWStorage, self).get_direct_upload_url(path, mime_type, requires_cors)
def complete_chunked_upload(self, uuid, final_path, storage_metadata):
self._initialize_cloud_conn()
# RadosGW does not support multipart copying from keys, so are forced to join
# it all locally and then reupload.
# See https://github.com/ceph/ceph/pull/5139
chunk_list = self._chunk_list_from_metadata(storage_metadata)
if len(chunk_list) == 1:
# If there's only one chunk, just "move" the key and call it a day.
chunk_path = chunk_list[0].path
self._cloud_bucket.copy_key(final_path, self._bucket_name, chunk_path)
self._cloud_bucket.delete_key(chunk_path)
return
concatenated = filelike.FilelikeStreamConcat(self._chunk_generator(chunk_list))
self.stream_write(final_path, concatenated)