Make sure the realm is connected before heartbeat checks start.

buildman/component/buildcomponent.py

@@ -18,6 +18,7 @@ from data.database import BUILD_PHASE
 
 HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
+INITIAL_TIMEOUT = 25
 
 LOGGER = logging.getLogger(__name__)
 
@@ -304,17 +305,14 @@ class BuildComponent(BaseComponent):
     """ Updates the last known heartbeat. """
     self._last_heartbeat = datetime.datetime.now()
 
-  def _start_heartbeat(self, loop):
-    """ Begins an async loop to keep a heartbeat going with a client. """
-    trollius.set_event_loop(loop)
-    loop.run_until_complete(self._heartbeat())
-
   @trollius.coroutine
   def _heartbeat(self):
     """ Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
         and updating the heartbeat in the build status dictionary (if applicable). This allows
         the build system to catch crashes from either end.
     """
+    yield From(trollius.sleep(INITIAL_TIMEOUT))
+
     while True:
       # If the component is no longer running or actively building, nothing more to do.
       if (self._component_status != ComponentStatus.RUNNING and
@@ -335,11 +333,7 @@ class BuildComponent(BaseComponent):
 
       # Check the heartbeat from the worker.
       LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
-      if not self._last_heartbeat:
-        self._timeout()
-        return
-
-      if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
+      if self._last_heartbeat and self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
         self._timeout()
         return
 
@@ -347,7 +341,7 @@ class BuildComponent(BaseComponent):
 
   def _timeout(self):
     self._set_status(ComponentStatus.TIMED_OUT)
-    LOGGER.warning('Build component (token "%s") timed out', self.expected_token)
+    LOGGER.warning('Build component with realm %s has timed out', self.builder_realm)
     self._dispose(timed_out=True)
 
   def _dispose(self, timed_out=False):