diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py index a1b7809c2..90cd1cd68 100644 --- a/buildman/manager/ephemeral.py +++ b/buildman/manager/ephemeral.py @@ -22,7 +22,7 @@ from util.morecollections import AttrDict logger = logging.getLogger(__name__) -ETCD_DISABLE_TIMEOUT = 0 +ETCD_MAX_WATCH_TIMEOUT = 30 EC2_API_TIMEOUT = 20 RETRY_IMMEDIATELY_TIMEOUT = 0 @@ -85,7 +85,7 @@ class EphemeralBuilderManager(BaseManager): '*' if recursive else '', existing_index, etcd_result) except ReadTimeoutError: - logger.debug('Read-timeout on etcd watch: %s', etcd_key) + logger.debug('Read-timeout on etcd watch %s, rescheduling', etcd_key) except (ProtocolError, etcd.EtcdException): logger.exception('Exception on etcd watch: %s', etcd_key) @@ -112,7 +112,7 @@ class EphemeralBuilderManager(BaseManager): '*' if recursive else '', start_index) watch_future = self._etcd_client.watch(etcd_key, recursive=recursive, index=start_index, - timeout=ETCD_DISABLE_TIMEOUT) + timeout=ETCD_MAX_WATCH_TIMEOUT) watch_future.add_done_callback(callback_wrapper) self._watch_tasks[watch_task_key] = async(watch_future)