Automatically disable build triggers with successive failures or internal errors

We allow users to reenable them manually once disabled

data/model/build.py

@@ -263,6 +263,45 @@ def toggle_build_trigger(trigger, enabled, reason='user_toggled'):
   trigger.enabled = enabled
 
   if not enabled:
-    trigger.disabled_reason = DisableReason.get(name=reason)
+    trigger.disabled_reason = RepositoryBuildTrigger.disabled_reason.get_id(reason)
 
   trigger.save()
+
+
+def update_trigger_disable_status(trigger, final_phase):
+  """ Updates the disable status of the given build trigger. If the build trigger had a
+      failure, then the counter is increased and, if we've reached the limit, the trigger is
+      automatically disabled. Otherwise, if the trigger succeeded, it's counter is reset. This
+      ensures that triggers that continue to error are eventually automatically disabled.
+  """
+  with db_transaction():
+    # If the build completed successfully, then reset the successive counters.
+    if final_phase == BUILD_PHASE.COMPLETE:
+      trigger.successive_failure_count = 0
+      trigger.successive_internal_error_count = 0
+      trigger.save()
+      return
+
+    # Otherwise, increment the counters and check for trigger disable.
+    if final_phase == BUILD_PHASE.ERROR:
+      trigger.successive_failure_count = trigger.successive_failure_count + 1
+      trigger.successive_internal_error_count = 0
+    elif final_phase == BUILD_PHASE.INTERNAL_ERROR:
+      trigger.successive_failure_count = 0
+      trigger.successive_internal_error_count = trigger.successive_internal_error_count + 1
+
+    # Check if we need to disable the trigger.    
+    failure_threshold = config.app_config.get('SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD')
+    error_threshold = config.app_config.get('SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD')
+
+    if failure_threshold and trigger.successive_failure_count >= failure_threshold:
+      trigger.enabled = False
+      trigger.disabled_reason = RepositoryBuildTrigger.disabled_reason.get_id('successive_build_failures')
+    elif (error_threshold and
+          trigger.successive_internal_error_count >= error_threshold):
+      trigger.enabled = False
+      trigger.disabled_reason = RepositoryBuildTrigger.disabled_reason.get_id('successive_build_internal_errors')
+
+    # Save the trigger changes.
+    trigger.save()
+  

data/model/test/test_build.py

@@ -0,0 +1,46 @@
+import pytest
+
+from mock import patch
+
+from data.database import BUILD_PHASE, RepositoryBuildTrigger
+from data.model.build import update_trigger_disable_status
+from test.fixtures import *
+
+TEST_FAIL_THRESHOLD = 5
+TEST_INTERNAL_ERROR_THRESHOLD = 2
+
+@pytest.mark.parametrize('starting_failure_count, starting_error_count, status, expected_reason', [
+  (0, 0, BUILD_PHASE.COMPLETE, None),
+  (10, 10, BUILD_PHASE.COMPLETE, None),
+
+  (TEST_FAIL_THRESHOLD - 1, TEST_INTERNAL_ERROR_THRESHOLD - 1, BUILD_PHASE.COMPLETE, None),
+  (TEST_FAIL_THRESHOLD - 1, 0, BUILD_PHASE.ERROR, 'successive_build_failures'),
+  (0, TEST_INTERNAL_ERROR_THRESHOLD - 1, BUILD_PHASE.INTERNAL_ERROR,
+   'successive_build_internal_errors'),
+])
+def test_update_trigger_disable_status(starting_failure_count, starting_error_count, status,
+                                       expected_reason, initialized_db):
+  test_config = {
+    'SUCCESSIVE_TRIGGER_FAILURE_DISABLE_THRESHOLD': TEST_FAIL_THRESHOLD,
+    'SUCCESSIVE_TRIGGER_INTERNAL_ERROR_DISABLE_THRESHOLD': TEST_INTERNAL_ERROR_THRESHOLD,
+  }
+
+  trigger = model.build.list_build_triggers('devtable', 'building')[0]
+  trigger.successive_failure_count = starting_failure_count
+  trigger.successive_internal_error_count = starting_error_count
+  trigger.enabled = True
+  trigger.save()
+
+  with patch('data.model.config.app_config', test_config):
+    update_trigger_disable_status(trigger, status)
+    updated_trigger = RepositoryBuildTrigger.get(uuid=trigger.uuid)
+
+    assert updated_trigger.enabled == (expected_reason is None)
+
+    if expected_reason is not None:
+      assert updated_trigger.disabled_reason.name == expected_reason
+    else:
+      assert updated_trigger.disabled_reason is None
+      assert updated_trigger.successive_failure_count == 0
+      assert updated_trigger.successive_internal_error_count == 0
+