Add a watchdog timer to the build worker to kill a build step that takes more than 20 minutes.

workers/worker.py

@@ -9,10 +9,12 @@ logger = logging.getLogger(__name__)
 
 
 class Worker(object):
-  def __init__(self, queue, poll_period_seconds=30, reservation_seconds=300):
+  def __init__(self, queue, poll_period_seconds=30, reservation_seconds=300,
+               watchdog_period_seconds=60):
     self._sched = Scheduler()
     self._poll_period_seconds = poll_period_seconds
     self._reservation_seconds = reservation_seconds
+    self._watchdog_period_seconds = watchdog_period_seconds
     self._stop = Event()
     self._queue = queue
 
@@ -20,6 +22,10 @@ class Worker(object):
     """ Return True if complete, False if it should be retried. """
     raise NotImplementedError('Workers must implement run.')
 
+  def watchdog(self):
+    """ Function that gets run once every watchdog_period_seconds. """
+    pass
+
   def poll_queue(self):
     logger.debug('Getting work item from queue.')
 
@@ -43,8 +49,8 @@ class Worker(object):
     logger.debug("Scheduling worker.")
 
     self._sched.start()
-    self._sched.add_interval_job(self.poll_queue,
-                                 seconds=self._poll_period_seconds)
+    self._sched.add_interval_job(self.poll_queue, seconds=self._poll_period_seconds)
+    self._sched.add_interval_job(self.watchdog, seconds=self._watchdog_period_seconds)
 
     while not self._stop.wait(1):
       pass