Add support to the build system for tracking if/when the build manager crashes and make sure builds are restarted within a few minutes

data/database.py

@@ -370,6 +370,7 @@ class RepositoryTag(BaseModel):
 class BUILD_PHASE(object):
   """ Build phases enum """
   ERROR = 'error'
+  INTERNAL_ERROR = 'internalerror'
   UNPACKING = 'unpacking'
   PULLING = 'pulling'
   BUILDING = 'building'

data/queue.py

@@ -120,10 +120,14 @@ class WorkQueue(object):
       self._currently_processing = False
 
   @staticmethod
-  def extend_processing(queue_item, seconds_from_now):
+  def extend_processing(queue_item, seconds_from_now, retry_count=None,
+                        minimum_extension=MINIMUM_EXTENSION):
     new_expiration = datetime.utcnow() + timedelta(seconds=seconds_from_now)
 
     # Only actually write the new expiration to the db if it moves the expiration some minimum
-    if new_expiration - queue_item.processing_expires > MINIMUM_EXTENSION:
+    if new_expiration - queue_item.processing_expires > minimum_extension:
+      if retry_count is not None:
+        queue_item.retries_remaining = retry_count
+
       queue_item.processing_expires = new_expiration
       queue_item.save()