Add executor-specific setup time support

This will allow us to make the setup time TTL for k8s-based builds much lower (on the order of a minute), which means faster timeouts and fallbacks (which is a better user experience).

buildman/manager/ephemeral.py

@@ -373,7 +373,7 @@ class EphemeralBuilderManager(BaseManager):
     # Load components for all realms currently known to the cluster
     async(self._register_existing_realms())
 
-  def setup_time(self):
+  def overall_setup_time(self):
     return self._ephemeral_setup_timeout
 
   def shutdown(self):
@@ -418,7 +418,6 @@ class EphemeralBuilderManager(BaseManager):
     realm = str(uuid.uuid4())
     token = str(uuid.uuid4())
     nonce = str(uuid.uuid4())
-    setup_time = self.setup_time()
 
     machine_max_expiration = self._manager_config.get('MACHINE_MAX_TIME', 7200)
     max_expiration = datetime.utcnow() + timedelta(seconds=machine_max_expiration)
@@ -526,6 +525,9 @@ class EphemeralBuilderManager(BaseManager):
     })
 
     try:
+      setup_time = started_with_executor.setup_time or self.overall_setup_time()
+      logger.debug('Writing job key for job %s using executor %s with ID %s and ttl %s', build_uuid,
+                   started_with_executor.name, execution_id, setup_time)
       yield From(self._etcd_client.write(self._etcd_realm_key(realm), realm_spec, prevExist=False,
                                          ttl=setup_time))
     except (KeyError, etcd.EtcdKeyError):

buildman/manager/executor.py

@@ -55,6 +55,13 @@ class BuilderExecutor(object):
     """ Name returns the unique name for this executor. """
     return self.executor_config.get('NAME') or self.__class__.__name__
 
+  @property
+  def setup_time(self):
+    """ Returns the amount of time (in seconds) to wait for the execution to start for the build.
+        If None, the manager's default will be used.
+    """
+    return self.executor_config.get('SETUP_TIME')
+
   @coroutine
   def start_builder(self, realm, token, build_uuid):
     """ Create a builder with the specified config. Returns a unique id which can be used to manage

buildman/server.py

@@ -168,7 +168,7 @@ class BuilderServer(object):
       logger.debug('Checking for more work for %d active workers',
                    self._lifecycle_manager.num_workers())
 
-      processing_time = self._lifecycle_manager.setup_time() + SETUP_LEEWAY_SECONDS
+      processing_time = self._lifecycle_manager.overall_setup_time() + SETUP_LEEWAY_SECONDS
       job_item = self._queue.get(processing_time=processing_time, ordering_required=True)
       if job_item is None:
         logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)