From a30b35870965e0ddaa8059705dc9f4134074daaa Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Fri, 28 Oct 2016 17:11:36 -0400 Subject: [PATCH] add staggered worker startup Fixes #787 --- config.py | 3 +++ workers/worker.py | 17 +++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/config.py b/config.py index 027f36254..5164396e4 100644 --- a/config.py +++ b/config.py @@ -382,3 +382,6 @@ class DefaultConfig(object): # Overridable list of reverse DNS prefixes that are reserved for internal use on labels. LABEL_KEY_RESERVED_PREFIXES = [] + + # Delays workers from starting until a random point in time between 0 and their regular interval. + STAGGERED_WORKERS = True diff --git a/workers/worker.py b/workers/worker.py index 19cfb903a..1ed3af6fa 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -3,14 +3,17 @@ import signal import sys import socket -from threading import Event -from apscheduler.schedulers.background import BackgroundScheduler from datetime import datetime, timedelta +from functools import wraps +from random import randint +from threading import Event + +from apscheduler.schedulers.background import BackgroundScheduler from raven import Client from app import app from data.database import UseThenDisconnect -from functools import wraps + logger = logging.getLogger(__name__) @@ -69,12 +72,14 @@ class Worker(object): logger.debug('Scheduling worker.') - soon = datetime.now() + timedelta(seconds=.001) - self._sched.start() for operation_func, operation_sec in self._operations: + start_date = datetime.now() + timedelta(seconds=0.001) + if app.config.get('STAGGER_WORKERS'): + start_date += timedelta(seconds=randint(1, operation_sec)) + logger.debug('First run scheduled for %s', start_date) self._sched.add_job(operation_func, 'interval', seconds=operation_sec, - start_date=soon, max_instances=1) + start_date=start_date, max_instances=1) self._setup_and_wait_for_shutdown()