vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Have Swift storage delete segments when deleting dynamic large objects

Browse source 
This ensures that we reclaim the space, rather than simply deleting the manifest

Fixes https://jira.coreos.com/browse/QUAY-942
This commit is contained in:
Joseph Schorr 2018-05-16 16:01:49 -04:00
parent 66b4e45929
commit 57523d22de
3 changed files with 105 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

31
storage/test/test_swift.py
View file

@ -50,13 +50,24 @@ class FakeSwift(object):
return 'http://fake/swift', None
def head_object(self, container, path):
return self.containers[container].get(path)
return self.containers.get(container, {}).get(path, {}).get('headers', None)
def copy_object(self, container, path, target):
pieces = target.split('/', 2)
_, content = self.get_object(container, path)
self.put_object(pieces[1], pieces[2], content)
def get_container(self, container, prefix=None, full_listing=None):
container_entries = self.containers[container]
objs = []
for path, data in list(container_entries.iteritems()):
if not prefix or path.startswith(prefix):
objs.append({
'name': path,
'bytes': len(data['content']),
})
return {}, objs
def put_object(self, container, path, content, chunk_size=None, content_type=None, headers=None):
if not isinstance(content, str):
if hasattr(content, 'read'):
@ -68,7 +79,7 @@ class FakeSwift(object):
'content': content,
'chunk_size': chunk_size,
'content_type': content_type,
'headers': headers or {},
'headers': headers or {'is': True},
}
digest = hashlib.md5()
@ -179,7 +190,7 @@ def test_copy_to():
assert another_swift.exists('somepath')
assert swift.get_content('somepath') == 'some content here'
assert another_swift.get_content('somepath') == 'some content here'
assert another_swift.get_content('somepath') == 'some content here'
def test_copy_to_different():
swift = FakeSwiftStorage(**base_args)
@ -197,7 +208,7 @@ def test_copy_to_different():
assert another_swift.exists('somepath')
assert swift.get_content('somepath') == 'some content here'
assert another_swift.get_content('somepath') == 'some content here'
assert another_swift.get_content('somepath') == 'some content here'
def test_checksum():
swift = FakeSwiftStorage(**base_args)
@ -238,6 +249,18 @@ def test_chunked_upload(chunks, max_chunk_size, read_until_end):
swift.complete_chunked_upload(uuid, 'somepath', metadata)
assert swift.get_content('somepath') == ''.join(chunks)
# Ensure each of the segments exist.
for segment in metadata['segments']:
assert swift.exists(segment.path)
# Delete the file and ensure all of its segments were removed.
swift.remove('somepath')
assert not swift.exists('somepath')
for segment in metadata['segments']:
assert not swift.exists(segment.path)
def test_cancel_chunked_upload():
swift = FakeSwiftStorage(**base_args)
uuid, metadata = swift.initiate_chunked_upload()