vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Automatically disable build triggers with successive failures or internal errors

Browse source 
We allow users to reenable them manually once disabled
This commit is contained in:
Joseph Schorr 2017-10-18 16:03:38 -04:00 • committed by Joseph Schorr
parent c35eec0615
commit 93d79e777e
9 changed files with 166 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

41
data/model/build.py
View file

@ -263,6 +263,45 @@ def toggle_build_trigger(trigger, enabled, reason='user_toggled'):
trigger.enabled = enabled
if not enabled:
trigger.disabled_reason = DisableReason.get(name=reason)
trigger.disabled_reason = RepositoryBuildTrigger.disabled_reason.get_id(reason)
trigger.save()
def update_trigger_disable_status(trigger, final_phase):
""" Updates the disable status of the given build trigger. If the build trigger had a
failure, then the counter is increased and, if we've reached the limit, the trigger is
automatically disabled. Otherwise, if the trigger succeeded, it's counter is reset. This
ensures that triggers that continue to error are eventually automatically disabled.
"""
with db_transaction():
# If the build completed successfully, then reset the successive counters.
if final_phase == BUILD_PHASE.COMPLETE:
trigger.successive_failure_count = 0
trigger.successive_internal_error_count = 0
trigger.save()
return
# Otherwise, increment the counters and check for trigger disable.
if final_phase == BUILD_PHASE.ERROR:
trigger.successive_failure_count = trigger.successive_failure_count + 1
trigger.successive_internal_error_count = 0
elif final_phase == BUILD_PHASE.INTERNAL_ERROR:
trigger.successive_failure_count = 0
trigger.successive_internal_error_count = trigger.successive_internal_error_count + 1
# Check if we need to disable the trigger.
failure_threshold = config.app_config.get('SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD')
error_threshold = config.app_config.get('SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD')
if failure_threshold and trigger.successive_failure_count >= failure_threshold:
trigger.enabled = False
trigger.disabled_reason = RepositoryBuildTrigger.disabled_reason.get_id('successive_build_failures')
elif (error_threshold and
trigger.successive_internal_error_count >= error_threshold):
trigger.enabled = False
trigger.disabled_reason = RepositoryBuildTrigger.disabled_reason.get_id('successive_build_internal_errors')
# Save the trigger changes.
trigger.save()