From 055a6b0c377f143038c1413b14f5e0137f143fdd Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 23 Dec 2014 11:18:10 -0500 Subject: [PATCH] Add a total maximum time that a machine is allowed to stick around before we terminate it more forcefully. --- buildman/manager/ephemeral.py | 13 +++++++++++-- test/test_buildman.py | 7 ++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py index fdc116e5b..7d9eacdc2 100644 --- a/buildman/manager/ephemeral.py +++ b/buildman/manager/ephemeral.py @@ -135,8 +135,12 @@ class EphemeralBuilderManager(BaseManager): ttl = self.setup_time() expiration = datetime.utcnow() + timedelta(seconds=ttl) + machine_max_expiration = self._manager_config.get('MACHINE_MAX_TIME', 7200) + max_expiration = datetime.utcnow() + timedelta(seconds=machine_max_expiration) + payload = { 'expiration': calendar.timegm(expiration.timetuple()), + 'max_expiration': calendar.timegm(max_expiration.timetuple()), } try: @@ -154,7 +158,7 @@ class EphemeralBuilderManager(BaseManager): # Store the builder in etcd associated with the job id payload['builder_id'] = builder_id - yield From(self._etcd_client.write(job_key, payload, prevExist=True)) + yield From(self._etcd_client.write(job_key, payload, prevExist=True, ttl=ttl)) raise Return(True) @@ -192,12 +196,17 @@ class EphemeralBuilderManager(BaseManager): job_key = self._etcd_job_key(build_job) build_job_response = yield From(self._etcd_client.read(job_key)) - ttl = self.heartbeat_period_sec * 2 + max_expiration = datetime.utcfromtimestamp(build_job_response.value['max_expiration']) + max_expiration_remaining = max_expiration - datetime.utcnow() + max_expiration_sec = max(0, int(max_expiration_remaining.total_seconds())) + + ttl = min(self.heartbeat_period_sec * 2, max_expiration_sec) new_expiration = datetime.utcnow() + timedelta(seconds=ttl) payload = { 'expiration': calendar.timegm(new_expiration.timetuple()), 'builder_id': build_job_response.value['builder_id'], + 'max_expiration': build_job_response.value['max_expiration'], } yield From(self._etcd_client.write(job_key, payload, ttl=ttl)) diff --git a/test/test_buildman.py b/test/test_buildman.py index d31539a3d..6835cdd49 100644 --- a/test/test_buildman.py +++ b/test/test_buildman.py @@ -159,8 +159,13 @@ class TestEphemeral(unittest.TestCase): @async_test def test_heartbeat_response(self): + expiration_timestamp = time.time() + 60 builder_result = Mock(spec=etcd.EtcdResult) - builder_result.value = {'builder_id': '123', 'expiration': '123'} + builder_result.value = { + 'builder_id': '123', + 'expiration': expiration_timestamp, + 'max_expiration': expiration_timestamp, + } self.etcd_client_mock.read = Mock(return_value=builder_result) yield From(self.manager.job_heartbeat(self.mock_job))