Handle internal errors in a better fashion: If a build would be marked as internal error, only do so if there are retries remaining. Otherwise, we mark it as failed (since it won't be rebuilt anyway)

buildman/component/buildcomponent.py

@@ -255,7 +255,8 @@ class BuildComponent(BaseComponent):
 
       # Write the error to the log.
       self._build_status.set_error(worker_error.public_message(), worker_error.extra_data(),
-                                   internal_error=worker_error.is_internal_error())
+                                   internal_error=worker_error.is_internal_error(),
+                                   requeued=self._current_job.has_retries_remaining())
 
       # Send the notification that the build has failed.
       self._current_job.send_notification('build_failure',
@@ -363,7 +364,8 @@ class BuildComponent(BaseComponent):
     # If we still have a running job, then it has not completed and we need to tell the parent
     # manager.
     if self._current_job is not None:
-      self._build_status.set_error('Build worker timed out', internal_error=True)
+      self._build_status.set_error('Build worker timed out', internal_error=True,
+                                   requeued=self._current_job.has_retries_remaining())
 
       self.parent_manager.job_completed(self._current_job, BuildJobResult.INCOMPLETE, self)
       self._build_status = None

buildman/jobutil/buildjob.py

@@ -25,6 +25,9 @@ class BuildJob(object):
           'Could not parse build queue item config with ID %s' % self.job_details['build_uuid']
       )
 
+  def has_retries_remaining(self):
+    return self.job_item.retries_remaining > 0
+
   def send_notification(self, kind, error_message=None):
     tags = self.build_config.get('docker_tags', ['latest'])
     event_data = {

buildman/jobutil/buildstatus.py

@@ -39,8 +39,8 @@ class StatusHandler(object):
     self._current_command = command
     self._append_log_message(command, self._build_logs.COMMAND, extra_data)
 
-  def set_error(self, error_message, extra_data=None, internal_error=False):
-    self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error else BUILD_PHASE.ERROR)
+  def set_error(self, error_message, extra_data=None, internal_error=False, requeued=False):
+    self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error and requeued else BUILD_PHASE.ERROR)
 
     extra_data = extra_data or {}
     extra_data['internal_error'] = internal_error