Add multiple executor and whitelist support to build manager
This commit is contained in:
Joseph Schorr
parent
6bdbe25cdc
commit
811413fe9c
4 changed files with 239 additions and 181 deletions
|
|
@ -59,7 +59,7 @@ class EphemeralBuilderManager(BaseManager):
|
|||
self._etcd_realm_prefix = None
|
||||
self._etcd_builder_prefix = None
|
||||
|
||||
self._etcd_lock_prefix = Nopne
|
||||
self._etcd_lock_prefix = None
|
||||
self._ephemeral_api_timeout = DEFAULT_EPHEMERAL_API_TIMEOUT
|
||||
|
||||
self._component_to_job = {}
|
||||
|
|
@ -205,6 +205,10 @@ class EphemeralBuilderManager(BaseManager):
|
|||
self._job_uuid_to_component[build_job.job_details['build_uuid']] = component
|
||||
return component
|
||||
|
||||
@property
|
||||
def registered_executors(self):
|
||||
return self._executors
|
||||
|
||||
@coroutine
|
||||
def _register_existing_realms(self):
|
||||
try:
|
||||
|
|
@ -227,16 +231,26 @@ class EphemeralBuilderManager(BaseManager):
|
|||
# no realms have been registered yet
|
||||
pass
|
||||
|
||||
def _load_executor(self, executor_class_name, executor_config):
|
||||
executor_klass = EXECUTORS.get(executor_class_name)
|
||||
if executor_klass is None:
|
||||
logger.error('Unknown executor %s; skipping install', executor_class_name)
|
||||
return
|
||||
|
||||
self._executors.append(executor_klass(executor_config, self.manager_hostname))
|
||||
|
||||
def initialize(self, manager_config):
|
||||
logger.debug('Calling initialize')
|
||||
self._manager_config = manager_config
|
||||
|
||||
# TODO(jschorr): We need to make this backwards compatible with existing config, as well as test(s)
|
||||
for config in manager_config.get('EXECUTORS', []):
|
||||
executor_klass = EXECUTORS.get(config['EXECUTOR'])
|
||||
executor_config = config.get('CONFIG', {})
|
||||
executor_config.update(manager_config.get('EXECUTOR_CONFIG', {}))
|
||||
self._executors.append(executor_klass(executor_config, self.manager_hostname))
|
||||
# Note: Executor config can be defined either as a single block of EXECUTOR_CONFIG (old style)
|
||||
# or as a new set of executor configurations, with the order determining how we fallback. We
|
||||
# check for both here to ensure backwards compatibility.
|
||||
if manager_config.get('EXECUTORS'):
|
||||
for executor_config in manager_config['EXECUTORS']:
|
||||
self._load_executor(executor_config.get('EXECUTOR'), executor_config)
|
||||
else:
|
||||
self._load_executor(manager_config.get('EXECUTOR'), manager_config.get('EXECUTOR_CONFIG'))
|
||||
|
||||
etcd_host = self._manager_config.get('ETCD_HOST', '127.0.0.1')
|
||||
etcd_port = self._manager_config.get('ETCD_PORT', 2379)
|
||||
|
|
@ -265,14 +279,14 @@ class EphemeralBuilderManager(BaseManager):
|
|||
restarter=self._register_existing_realms)
|
||||
|
||||
self._etcd_lock_prefix = self._manager_config.get('ETCD_LOCK_PREFIX', 'locks/')
|
||||
self._ephemeral_api_timeout = self._manager_config.get('API_TIMEOUT', DEFAULT_EPHEMERAL_API_TIMEOUT)
|
||||
self._ephemeral_api_timeout = self._manager_config.get('API_TIMEOUT',
|
||||
DEFAULT_EPHEMERAL_API_TIMEOUT)
|
||||
|
||||
# Load components for all realms currently known to the cluster
|
||||
async(self._register_existing_realms())
|
||||
|
||||
def setup_time(self):
|
||||
setup_time = self._manager_config.get('MACHINE_SETUP_TIME', 300)
|
||||
return setup_time
|
||||
return self._manager_config.get('MACHINE_SETUP_TIME', 300)
|
||||
|
||||
def shutdown(self):
|
||||
logger.debug('Shutting down worker.')
|
||||
|
|
@ -343,27 +357,43 @@ class EphemeralBuilderManager(BaseManager):
|
|||
logger.exception('Exception when writing job %s to etcd', build_uuid)
|
||||
raise Return(False, RETRY_IMMEDIATELY_TIMEOUT)
|
||||
|
||||
started = False
|
||||
started_with_executor = None
|
||||
logger.debug("executors are: %s", self._executors)
|
||||
|
||||
for executor in self._executors:
|
||||
# TODO(jschorr): gate on whitelist logic
|
||||
executor_type = executor.__class__.__name__
|
||||
|
||||
# Check if we can use this executor based on its whitelist, by namespace.
|
||||
namespace = build_job.namespace
|
||||
if not executor.allowed_for_namespace(namespace):
|
||||
logger.debug('Job %s (namespace: %s) cannot use executor %s', build_uuid, namespace,
|
||||
executor_type)
|
||||
continue
|
||||
|
||||
# Check if we can use this executor based on the retries remaining.
|
||||
if executor.minimum_retry_threshold > build_job.retries_remaining:
|
||||
logger.debug('Job %s cannot use executor %s due to not meeting retry threshold', build_uuid,
|
||||
executor_type)
|
||||
continue
|
||||
|
||||
logger.debug('Starting builder for job: %s with executor: %s', build_uuid, executor_type)
|
||||
|
||||
try:
|
||||
builder_id = yield From(executor.start_builder(realm, token, build_uuid))
|
||||
metric_queue.put_deprecated('EphemeralBuilderStarted', 1, unit='Count')
|
||||
metric_queue.ephemeral_build_workers.Inc(labelvalues=[builder_id, build_uuid])
|
||||
started = True
|
||||
started_with_executor = executor
|
||||
break
|
||||
except:
|
||||
logger.exception('Exception when starting builder for job: %s', build_uuid)
|
||||
continue
|
||||
|
||||
if not started:
|
||||
logger.error('Could not start any ephemeral workers.')
|
||||
if started_with_executor is None:
|
||||
logger.error('Could not start ephemeral worker for build %s', build_uuid)
|
||||
raise Return(False, self._ephemeral_api_timeout)
|
||||
|
||||
logger.debug('Started builder for job: %s with executor: %s', build_uuid, executor_type)
|
||||
|
||||
# Store the builder in etcd associated with the job id
|
||||
try:
|
||||
payload['builder_id'] = builder_id
|
||||
|
|
@ -392,8 +422,7 @@ class EphemeralBuilderManager(BaseManager):
|
|||
logger.exception('Exception when writing realm %s to etcd for job %s', realm, build_uuid)
|
||||
raise Return(False, setup_time)
|
||||
|
||||
self._job_to_executor[builder_id] = executor
|
||||
|
||||
self._job_to_executor[builder_id] = started_with_executor
|
||||
raise Return(True, None)
|
||||
|
||||
@coroutine
|
||||
|
|
|
|||
Reference in a new issue