vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix TTL on heartbeat in etcd

Browse source 
Until now, once the heartbeat has expired, we would issue a TTL that is negative, which causes etcd to either raise an exception or simply ignore the expiration (depending on the version of etcd). This change ensures that once the key is expired, it is removed immediately via a set of a TTL of 0. Also adds tests for this case and the normal expiration case.
This commit is contained in:
Joseph Schorr 2016-08-01 13:18:21 -04:00
parent abce6a8dbc
commit c29f9ccc7f
2 changed files with 24 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
buildman/manager/ephemeral.py
View file

@ -382,6 +382,7 @@ class EphemeralBuilderManager(BaseManager):
max_expiration = datetime.utcnow() + timedelta(seconds=machine_max_expiration)
payload = {
# TODO: remove expiration (but not max_expiration) after migration; not used.
'expiration': calendar.timegm(expiration.timetuple()),
'max_expiration': calendar.timegm(max_expiration.timetuple()),
'nonce': nonce,
@ -566,13 +567,17 @@ class EphemeralBuilderManager(BaseManager):
new_expiration = datetime.utcnow() + timedelta(seconds=ttl)
payload = {
# TODO: remove expiration (but not max_expiration) after migration; not used.
'expiration': calendar.timegm(new_expiration.timetuple()),
'job_queue_item': build_job.job_item,
'max_expiration': build_job_metadata['max_expiration'],
'had_heartbeat': True,
}
yield From(self._etcd_client.write(job_key, json.dumps(payload), ttl=ttl))
# Note: A TTL of < 0 in etcd results in the key *never being expired*. We use a max here
# to ensure that if the TTL is < 0, the key will expire immediately.
etcd_ttl = max(ttl, 0)
yield From(self._etcd_client.write(job_key, json.dumps(payload), ttl=etcd_ttl))
self.job_heartbeat_callback(build_job)
def _etcd_job_key(self, build_job):