57 lines
1.9 KiB
Python
57 lines
1.9 KiB
Python
|
import logging
|
||
|
|
import logging.config
|
||
|
|
|
||
|
|
from datetime import timedelta
|
||
|
|
|
||
|
|
from app import app, storage
|
||
|
|
from data.database import UseThenDisconnect, BlobUpload
|
||
|
|
from data import model
|
||
|
|
from workers.worker import Worker
|
||
|
|
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
DELETION_DATE_THRESHOLD = timedelta(days=2)
|
||
|
|
BLOBUPLOAD_CLEANUP_FREQUENCY = app.config.get('BLOBUPLOAD_CLEANUP_FREQUENCY', 60*60)
|
||
|
|
|
||
|
|
|
||
|
|
class BlobUploadCleanupWorker(Worker):
|
||
|
|
def __init__(self):
|
||
|
|
super(BlobUploadCleanupWorker, self).__init__()
|
||
|
|
self.add_operation(self._cleanup_uploads, BLOBUPLOAD_CLEANUP_FREQUENCY)
|
||
|
|
|
||
|
|
def _cleanup_uploads(self):
|
||
|
|
""" Performs garbage collection on the blobupload table. """
|
||
|
|
while True:
|
||
|
|
# Find all blob uploads older than the threshold (typically a week) and delete them.
|
||
|
|
with UseThenDisconnect(app.config):
|
||
|
|
stale_upload = model.blob.get_stale_blob_upload(DELETION_DATE_THRESHOLD)
|
||
|
|
if stale_upload is None:
|
||
|
|
logger.debug('No additional stale blob uploads found')
|
||
|
|
return
|
||
|
|
|
||
|
|
# Remove the stale upload from storage.
|
||
|
|
logger.debug('Removing stale blob upload %s', stale_upload.uuid)
|
||
|
|
try:
|
||
|
|
storage.cancel_chunked_upload([stale_upload.location.name], stale_upload.uuid,
|
||
|
|
stale_upload.storage_metadata)
|
||
|
|
except Exception as ex:
|
||
|
|
logger.debug('Got error when trying to cancel chunked upload %s: %s', stale_upload.uuid,
|
||
|
|
ex.message)
|
||
|
|
|
||
|
|
# Delete the stale upload's row.
|
||
|
|
with UseThenDisconnect(app.config):
|
||
|
|
try:
|
||
|
|
stale_upload.delete_instance()
|
||
|
|
except BlobUpload.DoesNotExist:
|
||
|
|
continue
|
||
|
|
|
||
|
|
logger.debug('Removed stale blob upload %s', stale_upload.uuid)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
|
||
|
|
worker = BlobUploadCleanupWorker()
|
||
|
|
worker.start()
|