Add support to the build system for tracking if/when the build manager crashes and make sure builds are restarted within a few minutes

endpoints/api/build.py

@@ -1,5 +1,7 @@
 import logging
 import json
+import time
+import datetime
 
 from flask import request, redirect
 
@@ -9,7 +11,7 @@ from endpoints.api import (RepositoryParamResource, parse_args, query_param, nic
                            ApiResource, internal_only, format_date, api, Unauthorized, NotFound)
 from endpoints.common import start_build
 from endpoints.trigger import BuildTrigger
-from data import model
+from data import model, database
 from auth.auth_context import get_authenticated_user
 from auth.permissions import ModifyRepositoryPermission, AdministerOrganizationPermission
 from data.buildlogs import BuildStatusRetrievalError
@@ -65,6 +67,13 @@ def build_status_view(build_obj, can_write=False):
     status = {}
     phase = 'cannot_load'
 
+  # If the status contains a heartbeat, then check to see if has been written in the last few
+  # minutes. If not, then the build timed out.
+  if status is not None and 'heartbeat' in status and status['heartbeat']:
+    heartbeat = datetime.datetime.fromtimestamp(status['heartbeat'])
+    if datetime.datetime.now() - heartbeat > datetime.timedelta(minutes=1):
+      phase = database.BUILD_PHASE.INTERNAL_ERROR
+
   logger.debug('Can write: %s job_config: %s', can_write, build_obj.job_config)
   resp = {
     'id': build_obj.uuid,