vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add a chunk cleanup queue for async GC of empty chunks

Browse source 
Instead of having the Swift storage engine try to delete the empty chunk(s) synchronously, we simply queue them and have a worker come along after 30s to delete the empty chunks. This has a few key benefits: it is async (doesn't slow down the push code), helps deal with Swift's eventual consistency (less retries necessary) and is generic for other storage engines if/when they need this as well
This commit is contained in:
Joseph Schorr 2016-11-10 13:54:04 -05:00
parent 59cb6bd216
commit 5f99448adc
12 changed files with 191 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

11
test/test_cloud_storage.py
View file

@ -3,7 +3,7 @@ import moto
import boto
import os
from storage import S3Storage
from storage import S3Storage, StorageContext
from storage.cloud import _CloudStorage, _PartUploadMetadata
from storage.cloud import _CHUNKS_KEY
from StringIO import StringIO
@ -13,6 +13,7 @@ _TEST_BUCKET = 'some_bucket'
_TEST_USER = 'someuser'
_TEST_PASSWORD = 'somepassword'
_TEST_PATH = 'some/cool/path'
_TEST_CONTEXT = StorageContext('nyc', None, None)
class TestCloudStorage(unittest.TestCase):
def setUp(self):
@ -21,7 +22,7 @@ class TestCloudStorage(unittest.TestCase):
# Create a test bucket and put some test content.
boto.connect_s3().create_bucket(_TEST_BUCKET)
self.engine = S3Storage(None, 'some/path', _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
self.engine = S3Storage(_TEST_CONTEXT, 'some/path', _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
self.engine.put_content(_TEST_PATH, _TEST_CONTENT)
def tearDown(self):
@ -51,7 +52,8 @@ class TestCloudStorage(unittest.TestCase):
def test_copy_samecreds(self):
# Copy the content to another engine.
another_engine = S3Storage(None, 'another/path', _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
another_engine = S3Storage(_TEST_CONTEXT, 'another/path', _TEST_BUCKET, _TEST_USER,
_TEST_PASSWORD)
self.engine.copy_to(another_engine, _TEST_PATH)
# Verify it can be retrieved.
@ -59,7 +61,8 @@ class TestCloudStorage(unittest.TestCase):
def test_copy_differentcreds(self):
# Copy the content to another engine.
another_engine = S3Storage(None, 'another/path', 'another_bucket', 'blech', 'password')
another_engine = S3Storage(_TEST_CONTEXT, 'another/path', 'another_bucket', 'blech',
'password')
boto.connect_s3().create_bucket('another_bucket')
self.engine.copy_to(another_engine, _TEST_PATH)