Add support to the build system for tracking if/when the build manager crashes and make sure builds are restarted within a few minutes

buildman/manager/basemanager.py

@@ -1,10 +1,23 @@
 class BaseManager(object):
   """ Base for all worker managers. """
-  def __init__(self, register_component, unregister_component, job_complete_callback):
+  def __init__(self, register_component, unregister_component, job_heartbeat_callback,
+               job_complete_callback):
     self.register_component = register_component
     self.unregister_component = unregister_component
+    self.job_heartbeat_callback = job_heartbeat_callback
     self.job_complete_callback = job_complete_callback
 
+  def job_heartbeat(self, build_job):
+    """ Method invoked to tell the manager that a job is still running. This method will be called
+        every few minutes. """
+    self.job_heartbeat_callback(build_job)
+
+  def setup_time(self):
+    """ Returns the number of seconds that the build system should wait before allowing the job
+        to be picked up again after called 'schedule'.
+    """
+    raise NotImplementedError
+
   def shutdown(self):
     """ Indicates that the build controller server is in a shutdown state and that no new jobs
         or workers should be performed. Existing workers should be cleaned up once their jobs