vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Make sure the realm is connected before heartbeat checks start.

Browse source
This commit is contained in:
Joseph Schorr 2014-11-26 17:02:49 -05:00
parent 52b7896835
commit a8473db87f
1 changed files with 5 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
buildman/component/buildcomponent.py
View file

@ -18,6 +18,7 @@ from data.database import BUILD_PHASE
HEARTBEAT_DELTA = datetime.timedelta(seconds=30) HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
HEARTBEAT_TIMEOUT = 10 HEARTBEAT_TIMEOUT = 10
INITIAL_TIMEOUT = 25
LOGGER = logging.getLogger(__name__) LOGGER = logging.getLogger(__name__)
@ -304,17 +305,14 @@ class BuildComponent(BaseComponent):
""" Updates the last known heartbeat. """ """ Updates the last known heartbeat. """
self._last_heartbeat = datetime.datetime.now() self._last_heartbeat = datetime.datetime.now()
def _start_heartbeat(self, loop):
""" Begins an async loop to keep a heartbeat going with a client. """
trollius.set_event_loop(loop)
loop.run_until_complete(self._heartbeat())
@trollius.coroutine @trollius.coroutine
def _heartbeat(self): def _heartbeat(self):
""" Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat """ Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
and updating the heartbeat in the build status dictionary (if applicable). This allows and updating the heartbeat in the build status dictionary (if applicable). This allows
the build system to catch crashes from either end. the build system to catch crashes from either end.
""" """
yield From(trollius.sleep(INITIAL_TIMEOUT))
while True: while True:
# If the component is no longer running or actively building, nothing more to do. # If the component is no longer running or actively building, nothing more to do.
if (self._component_status != ComponentStatus.RUNNING and if (self._component_status != ComponentStatus.RUNNING and
@ -335,11 +333,7 @@ class BuildComponent(BaseComponent):
# Check the heartbeat from the worker. # Check the heartbeat from the worker.
LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm) LOGGER.debug('Checking heartbeat on realm %s', self.builder_realm)
if not self._last_heartbeat: if self._last_heartbeat and self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
self._timeout()
return
if self._last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
self._timeout() self._timeout()
return return
@ -347,7 +341,7 @@ class BuildComponent(BaseComponent):
def _timeout(self): def _timeout(self):
self._set_status(ComponentStatus.TIMED_OUT) self._set_status(ComponentStatus.TIMED_OUT)
LOGGER.warning('Build component (token "%s") timed out', self.expected_token) LOGGER.warning('Build component with realm %s has timed out', self.builder_realm)
self._dispose(timed_out=True) self._dispose(timed_out=True)
def _dispose(self, timed_out=False): def _dispose(self, timed_out=False):