5f99448adc
Instead of having the Swift storage engine try to delete the empty chunk(s) synchronously, we simply queue them and have a worker come along after 30s to delete the empty chunks. This has a few key benefits: it is async (doesn't slow down the push code), helps deal with Swift's eventual consistency (less retries necessary) and is generic for other storage engines if/when they need this as well
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
import logging
|
|
import time
|
|
|
|
from app import app, storage, chunk_cleanup_queue
|
|
from workers.queueworker import QueueWorker, JobException
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
POLL_PERIOD_SECONDS = 10
|
|
|
|
|
|
class ChunkCleanupWorker(QueueWorker):
|
|
""" Worker which cleans up chunks enqueued by the storage engine(s). This is typically used to
|
|
cleanup empty chunks which are no longer needed.
|
|
"""
|
|
def process_queue_item(self, job_details):
|
|
logger.debug('Got chunk cleanup queue item: %s', job_details)
|
|
storage_location = job_details['location']
|
|
storage_path = job_details['path']
|
|
|
|
try:
|
|
storage.remove([storage_location], storage_path)
|
|
except IOError:
|
|
raise JobException()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False)
|
|
|
|
engines = set([config[0] for config in app.config.get('DISTRIBUTED_STORAGE_CONFIG', {}).values()])
|
|
if 'SwiftStorage' not in engines:
|
|
logger.debug('Swift storage not detected; sleeping')
|
|
while True:
|
|
time.sleep(10000)
|
|
|
|
logger.debug('Starting chunk cleanup worker')
|
|
worker = ChunkCleanupWorker(chunk_cleanup_queue, poll_period_seconds=POLL_PERIOD_SECONDS)
|
|
worker.start()
|