From 5e1a117ff37cdde742f7e7a6e74fdb4ec2cbbaf5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 16 Aug 2016 16:33:43 -0400 Subject: [PATCH 1/2] Delete the job first to prevent Kubernetes from starting another pod --- buildman/manager/executor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py index 82a438fb5..a3e3c24c3 100644 --- a/buildman/manager/executor.py +++ b/buildman/manager/executor.py @@ -408,6 +408,12 @@ class KubernetesExecutor(BuilderExecutor): def stop_builder(self, builder_id): pods_path = '/api/v1/namespaces/%s/pods' % self.namespace + # Delete the job itself. + try: + yield From(self._request('DELETE', self._job_path(builder_id))) + except: + logger.exception('Failed to send delete job call for job %s', builder_id) + # Delete the pod(s) for the job. selectorString = "job-name=%s" % builder_id try: @@ -415,12 +421,6 @@ class KubernetesExecutor(BuilderExecutor): except: logger.exception("Failed to send delete pod call for job %s", builder_id) - # Delete the job itself. - try: - yield From(self._request('DELETE', self._job_path(builder_id))) - except: - logger.exception('Failed to send delete job call for job %s', builder_id) - class LogPipe(threading.Thread): """ Adapted from http://codereview.stackexchange.com/a/17959 From 433b1575311f1c754e79bdb7f62fbfad5b54ce57 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 16 Aug 2016 16:38:48 -0400 Subject: [PATCH 2/2] Add extra check to ensure a build cannot be started without on_ready called --- buildman/component/buildcomponent.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index e8a80ca19..3c91443d9 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -70,13 +70,15 @@ class BuildComponent(BaseComponent): yield From(self._set_status(ComponentStatus.WAITING)) - def is_ready(self): - """ Determines whether a build component is ready to begin a build. """ - return self._component_status == ComponentStatus.RUNNING - @trollius.coroutine def start_build(self, build_job): """ Starts a build. """ + if self._component_status != ComponentStatus.RUNNING: + logger.debug('Could not start build for component %s (build %s, worker version: %s): %s', + self.builder_realm, build_job.repo_build.uuid, self._worker_version, + self._component_status) + raise Return() + logger.debug('Starting build for component %s (build %s, worker version: %s)', self.builder_realm, build_job.repo_build.uuid, self._worker_version)