From ef41e57aad6c8070b9c6368782d172be55e2c42e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 7 Nov 2016 15:45:15 -0500 Subject: [PATCH] Add executor-specific setup time support This will allow us to make the setup time TTL for k8s-based builds much lower (on the order of a minute), which means faster timeouts and fallbacks (which is a better user experience). --- buildman/manager/ephemeral.py | 6 ++++-- buildman/manager/executor.py | 7 +++++++ buildman/server.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py index f9e05404e..5071b9b07 100644 --- a/buildman/manager/ephemeral.py +++ b/buildman/manager/ephemeral.py @@ -373,7 +373,7 @@ class EphemeralBuilderManager(BaseManager): # Load components for all realms currently known to the cluster async(self._register_existing_realms()) - def setup_time(self): + def overall_setup_time(self): return self._ephemeral_setup_timeout def shutdown(self): @@ -418,7 +418,6 @@ class EphemeralBuilderManager(BaseManager): realm = str(uuid.uuid4()) token = str(uuid.uuid4()) nonce = str(uuid.uuid4()) - setup_time = self.setup_time() machine_max_expiration = self._manager_config.get('MACHINE_MAX_TIME', 7200) max_expiration = datetime.utcnow() + timedelta(seconds=machine_max_expiration) @@ -526,6 +525,9 @@ class EphemeralBuilderManager(BaseManager): }) try: + setup_time = started_with_executor.setup_time or self.overall_setup_time() + logger.debug('Writing job key for job %s using executor %s with ID %s and ttl %s', build_uuid, + started_with_executor.name, execution_id, setup_time) yield From(self._etcd_client.write(self._etcd_realm_key(realm), realm_spec, prevExist=False, ttl=setup_time)) except (KeyError, etcd.EtcdKeyError): diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py index 1f25dddb2..0dee78439 100644 --- a/buildman/manager/executor.py +++ b/buildman/manager/executor.py @@ -55,6 +55,13 @@ class BuilderExecutor(object): """ Name returns the unique name for this executor. """ return self.executor_config.get('NAME') or self.__class__.__name__ + @property + def setup_time(self): + """ Returns the amount of time (in seconds) to wait for the execution to start for the build. + If None, the manager's default will be used. + """ + return self.executor_config.get('SETUP_TIME') + @coroutine def start_builder(self, realm, token, build_uuid): """ Create a builder with the specified config. Returns a unique id which can be used to manage diff --git a/buildman/server.py b/buildman/server.py index f04060351..48806e21e 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -168,7 +168,7 @@ class BuilderServer(object): logger.debug('Checking for more work for %d active workers', self._lifecycle_manager.num_workers()) - processing_time = self._lifecycle_manager.setup_time() + SETUP_LEEWAY_SECONDS + processing_time = self._lifecycle_manager.overall_setup_time() + SETUP_LEEWAY_SECONDS job_item = self._queue.get(processing_time=processing_time, ordering_required=True) if job_item is None: logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)