add staggered worker startup

Fixes #787

config.py

@@ -382,3 +382,6 @@ class DefaultConfig(object):
 
   # Overridable list of reverse DNS prefixes that are reserved for internal use on labels.
   LABEL_KEY_RESERVED_PREFIXES = []
+
+  # Delays workers from starting until a random point in time between 0 and their regular interval.
+  STAGGERED_WORKERS = True

workers/worker.py

@@ -3,14 +3,17 @@ import signal
 import sys
 import socket
 
-from threading import Event
-from apscheduler.schedulers.background import BackgroundScheduler
 from datetime import datetime, timedelta
+from functools import wraps
+from random import randint
+from threading import Event
+
+from apscheduler.schedulers.background import BackgroundScheduler
 from raven import Client
 
 from app import app
 from data.database import UseThenDisconnect
-from functools import wraps
+
 
 logger = logging.getLogger(__name__)
 
@@ -69,12 +72,14 @@ class Worker(object):
 
     logger.debug('Scheduling worker.')
 
-    soon = datetime.now() + timedelta(seconds=.001)
-
     self._sched.start()
     for operation_func, operation_sec in self._operations:
+      start_date = datetime.now() + timedelta(seconds=0.001)
+      if app.config.get('STAGGER_WORKERS'):
+        start_date += timedelta(seconds=randint(1, operation_sec))
+      logger.debug('First run scheduled for %s', start_date)
       self._sched.add_job(operation_func, 'interval', seconds=operation_sec,
-                          start_date=soon, max_instances=1)
+                          start_date=start_date, max_instances=1)
 
 
     self._setup_and_wait_for_shutdown()