vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Better error handling for the build manager

Browse source 
Fixes #604
This commit is contained in:
Joseph Schorr 2015-10-06 17:45:02 -04:00
parent 57675487cb
commit 0f37e66cc8
2 changed files with 16 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
buildman/manager/ephemeral.py
View file

@ -99,6 +99,10 @@ class EphemeralBuilderManager(BaseManager):
if restarter is not None:
async(restarter())
except (KeyError, etcd.EtcdKeyError):
logger.debug('Etcd key already cleared: %s', etcd_key)
return
except etcd.EtcdException as eex:
# TODO(jschorr): This is a quick and dirty hack and should be replaced
# with a proper exception check.
@ -401,7 +405,7 @@ class EphemeralBuilderManager(BaseManager):
try:
yield From(self._etcd_client.delete(job_key))
except (KeyError, etcd.EtcdKeyError):
logger.exception('Builder is asking for job to be removed, but work already completed')
logger.debug('Builder is asking for job to be removed, but work already completed')
self.job_complete_callback(build_job, job_status)

13
buildman/manager/executor.py
View file

@ -160,8 +160,17 @@ class EC2Executor(BuilderExecutor):
@coroutine
def stop_builder(self, builder_id):
ec2_conn = self._get_conn()
terminated_instances = yield From(ec2_conn.terminate_instances([builder_id]))
try:
ec2_conn = self._get_conn()
terminated_instances = yield From(ec2_conn.terminate_instances([builder_id]))
except boto.exception.EC2ResponseError as ec2e:
if ec2e.error_code == 404:
logger.debug('Instance %s already terminated', builder_id)
return
logger.exception('Exception when trying to terminate instance %s', builder_id)
raise
if builder_id not in [si.id for si in terminated_instances]:
raise ExecutorException('Unable to terminate instance: %s' % builder_id)