vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add additional metrics on executor start and failure

Browse source 
This will allow us to register a pager if one of the executors starts failing consistently
This commit is contained in:
Joseph Schorr 2017-11-27 11:52:37 +02:00
parent 01bedf6150
commit 400a5db719
2 changed files with 17 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

13
buildman/manager/ephemeral.py
View file

@ -496,9 +496,22 @@ class EphemeralBuilderManager(BaseManager):
try:
execution_id = yield From(executor.start_builder(realm, token, build_uuid))
except:
try:
metric_queue.build_start_failure.Inc(labelvalues=[executor.name])
metric_queue.put_deprecated(('ExecutorFailure-%s' % executor.name), 1, unit='Count')
except:
logger.exception('Exception when writing failure metric for execution %s for job %s',
execution_id, build_uuid)
logger.exception('Exception when starting builder for job: %s', build_uuid)
continue
try:
metric_queue.build_start_success.Inc(labelvalues=[executor.name])
except:
logger.exception('Exception when writing success metric for execution %s for job %s',
execution_id, build_uuid)
try:
metric_queue.ephemeral_build_workers.Inc()
except: