vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #607 from coreos-inc/buildmanerrorhandle

Browse source 
Better error handling for the build manager
This commit is contained in:
josephschorr 2015-10-13 17:52:01 -04:00
commit e31dda35df
2 changed files with 16 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
buildman/manager/ephemeral.py
View file

@ -99,6 +99,10 @@ class EphemeralBuilderManager(BaseManager):
if restarter is not None: if restarter is not None:
async(restarter()) async(restarter())
except (KeyError, etcd.EtcdKeyError):
logger.debug('Etcd key already cleared: %s', etcd_key)
return
except etcd.EtcdException as eex: except etcd.EtcdException as eex:
# TODO(jschorr): This is a quick and dirty hack and should be replaced # TODO(jschorr): This is a quick and dirty hack and should be replaced
# with a proper exception check. # with a proper exception check.
@ -401,7 +405,7 @@ class EphemeralBuilderManager(BaseManager):
try: try:
yield From(self._etcd_client.delete(job_key)) yield From(self._etcd_client.delete(job_key))
except (KeyError, etcd.EtcdKeyError): except (KeyError, etcd.EtcdKeyError):
logger.exception('Builder is asking for job to be removed, but work already completed') logger.debug('Builder is asking for job to be removed, but work already completed')
self.job_complete_callback(build_job, job_status) self.job_complete_callback(build_job, job_status)

9
buildman/manager/executor.py
View file

@ -160,8 +160,17 @@ class EC2Executor(BuilderExecutor):
@coroutine @coroutine
def stop_builder(self, builder_id): def stop_builder(self, builder_id):
try:
ec2_conn = self._get_conn() ec2_conn = self._get_conn()
terminated_instances = yield From(ec2_conn.terminate_instances([builder_id])) terminated_instances = yield From(ec2_conn.terminate_instances([builder_id]))
except boto.exception.EC2ResponseError as ec2e:
if ec2e.error_code == 404:
logger.debug('Instance %s already terminated', builder_id)
return
logger.exception('Exception when trying to terminate instance %s', builder_id)
raise
if builder_id not in [si.id for si in terminated_instances]: if builder_id not in [si.id for si in terminated_instances]:
raise ExecutorException('Unable to terminate instance: %s' % builder_id) raise ExecutorException('Unable to terminate instance: %s' % builder_id)