vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Change logs worker to use a global lock in the inner loop and move storage out of the transaction

Browse source
This commit is contained in:
Joseph Schorr 2016-03-24 14:04:52 -04:00
parent 4aa079e743
commit dc8f9713f8
3 changed files with 86 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

60
workers/logrotateworker.py
View file

@ -11,49 +11,58 @@ from data.model import db_transaction
from data.model.log import (get_stale_logs, get_stale_logs_start_id,
get_stale_logs_cutoff_id, delete_stale_logs)
from util.registry.gzipwrap import GzipWrap
from workers.globalworkerbase import GlobalWorker
from util.locking import GlobalLock
from workers.worker import Worker
logger = logging.getLogger(__name__)
WORKER_FREQUENCY = 3600 * 6
STALE_AFTER = timedelta(days=30)
MIN_LOGS_PER_ROTATION = 10000
WORKER_FREQUENCY = app.config.get('ACTION_LOG_ROTATION_FREQUENCY', 3600 * 6)
SAVE_PATH = app.config.get('ACTION_LOG_ARCHIVE_PATH')
SAVE_LOCATION = app.config.get('ACTION_LOG_ARCHIVE_LOCATION')
class LogRotateWorker(GlobalWorker):
class LogRotateWorker(Worker):
""" Worker used to rotate old logs out the database and into storage. """
def __init__(self):
super(LogRotateWorker, self).__init__(app, sleep_period_seconds=WORKER_FREQUENCY)
super(LogRotateWorker, self).__init__()
self.add_operation(self._archive_logs, WORKER_FREQUENCY)
def perform_global_work(self):
def _archive_logs(self):
logger.debug('Attempting to rotate log entries')
while True:
with UseThenDisconnect(app.config):
with db_transaction():
cutoff_date = datetime.now() - STALE_AFTER
start_id = get_stale_logs_start_id()
cutoff_id = get_stale_logs_cutoff_id(cutoff_date)
with GlobalLock('ACTION_LOG_ROTATION') as gl:
if not gl:
logger.debug('Could not acquire global lock; sleeping')
return
if start_id is None or cutoff_id is None:
logger.warning('No logs to be archived.')
return
with UseThenDisconnect(app.config):
with db_transaction():
cutoff_date = datetime.now() - STALE_AFTER
start_id = get_stale_logs_start_id()
cutoff_id = get_stale_logs_cutoff_id(cutoff_date)
logger.debug('Found starting ID %s and cutoff ID %s', start_id, cutoff_id)
if start_id is None or cutoff_id is None:
logger.warning('No logs to be archived.')
return
approx_count = cutoff_id - start_id
if approx_count < MIN_LOGS_PER_ROTATION:
logger.debug('Not enough stale logs to warrant rotation (approx %d)', approx_count)
return
logger.debug('Found starting ID %s and cutoff ID %s', start_id, cutoff_id)
end_id = start_id + MIN_LOGS_PER_ROTATION
logs = (pretty_print_in_json(log)
for log in get_stale_logs(start_id, end_id))
approx_count = cutoff_id - start_id
if approx_count < MIN_LOGS_PER_ROTATION:
logger.debug('Not enough stale logs to warrant rotation (approx %d)', approx_count)
return
logger.debug('Archiving logs from IDs %s to %s', start_id, end_id)
filename = '%s%d-%d.txt.gz' % (SAVE_PATH, start_id, end_id)
storage.stream_write(SAVE_LOCATION, filename, GzipWrap(logs))
end_id = start_id + MIN_LOGS_PER_ROTATION
logs = (pretty_print_in_json(log)
for log in get_stale_logs(start_id, end_id))
logger.debug('Archiving logs from IDs %s to %s', start_id, end_id)
filename = '%s%d-%d.txt.gz' % (SAVE_PATH, start_id, end_id)
storage.stream_write(SAVE_LOCATION, filename, GzipWrap(logs))
with UseThenDisconnect(app.config):
delete_stale_logs(start_id, end_id)
@ -69,7 +78,10 @@ def pretty_print_in_json(log):
def main():
logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
if not features.ACTION_LOG_ROTATION or None in [SAVE_PATH, SAVE_LOCATION]:
logger.debug('Action log rotation worker not enabled; skipping')
while True:
time.sleep(100000)