Track whether builders ever came online in etcd. Mark builds which never successfully heartbeated as incomplete.

test/test_buildman.py

@@ -5,7 +5,7 @@ import time
 import json
 
 from trollius import coroutine, get_event_loop, From, Future, sleep, Return
-from mock import Mock
+from mock import Mock, ANY
 from threading import Event
 from urllib3.exceptions import ReadTimeoutError
 
@@ -191,13 +191,36 @@ class TestEphemeral(unittest.TestCase):
     expired_result._prev_node = Mock(spec=etcd.EtcdResult)
     expired_result._prev_node.value = json.dumps({'builder_id': '1234'})
 
-    self.manager._handle_builder_expiration(expired_result)
-
-    yield From(sleep(.01))
+    yield From(self.manager._handle_builder_expiration(expired_result))
 
     self.test_executor.stop_builder.assert_called_once_with('1234')
     self.assertEqual(self.test_executor.stop_builder.call_count, 1)
 
+  @async_test
+  def test_builder_never_starts(self):
+    test_component = yield From(self._setup_job_for_managers())
+
+    # Test that we are watching before anything else happens
+    self.etcd_client_mock.watch.assert_any_call('building/', recursive=True, timeout=0)
+
+    # Send a signal to the callback that a worker has expired
+    expired_result = Mock(spec=etcd.EtcdResult)
+    expired_result.action = EtcdAction.EXPIRE
+    expired_result.key = self.mock_job_key
+    expired_result._prev_node = Mock(spec=etcd.EtcdResult)
+    expired_result._prev_node.value = json.dumps({
+      'builder_id': '1234',
+      'had_heartbeat': False,
+      'job_queue_item': self.mock_job.job_item,
+    })
+
+    yield From(self.manager._handle_builder_expiration(expired_result))
+
+    self.test_executor.stop_builder.assert_called_once_with('1234')
+    self.assertEqual(self.test_executor.stop_builder.call_count, 1)
+
+    self.job_complete_callback.assert_called_once_with(ANY, BuildJobResult.INCOMPLETE)
+
   @async_test
   def test_change_worker(self):
     # Send a signal to the callback that a worker key has been changed
@@ -233,3 +256,4 @@ class TestEphemeral(unittest.TestCase):
 
 if __name__ == '__main__':
   unittest.main()
+