Add better errors if Redis is down

Fixes #1497

data/buildlogs.py

@@ -61,8 +61,8 @@ class RedisBuildLogs(object):
       llen = self._redis.llen(self._logs_key(build_id))
       log_entries = self._redis.lrange(self._logs_key(build_id), start_index, -1)
       return (llen, (json.loads(entry) for entry in log_entries))
-    except redis.ConnectionError:
-      raise BuildStatusRetrievalError('Cannot retrieve build logs')
+    except redis.ConnectionError as ce:
+      raise BuildStatusRetrievalError('Cannot retrieve build logs: %s' % ce)
 
   def expire_log_entries(self, build_id):
     """
@@ -87,8 +87,8 @@ class RedisBuildLogs(object):
     """
     try:
       fetched = self._redis.get(self._status_key(build_id))
-    except redis.ConnectionError:
-      raise BuildStatusRetrievalError('Cannot retrieve build status')
+    except redis.ConnectionError as ce:
+      raise BuildStatusRetrievalError('Cannot retrieve build status: %s' % ce)
 
     return json.loads(fetched) if fetched else None
 

endpoints/api/build.py

@@ -8,11 +8,11 @@ import hashlib
 from flask import request
 from rfc3987 import parse as uri_parse
 
-from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue
+from app import userfiles as user_files, build_logs, log_archive, dockerfile_build_queue
 from buildtrigger.basehandler import BuildTriggerHandler
 from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
                            require_repo_read, require_repo_write, validate_json_request,
-                           ApiResource, internal_only, format_date, api, path_param, 
+                           ApiResource, internal_only, format_date, api, path_param,
                            require_repo_admin)
 from endpoints.exception import Unauthorized, NotFound, InvalidRequest
 from endpoints.building import start_build, PreparedBuild
@@ -20,7 +20,8 @@ from data import database
 from data import model
 from auth.auth_context import get_authenticated_user
 from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
-                              AdministerRepositoryPermission, AdministerOrganizationPermission)
+                              AdministerRepositoryPermission, AdministerOrganizationPermission,
+                              SuperUserPermission)
 
 from data.buildlogs import BuildStatusRetrievalError
 from util.names import parse_robot_username
@@ -87,26 +88,33 @@ def trigger_view(trigger, can_read=False, can_admin=False, for_build=False):
 
 def build_status_view(build_obj):
   phase = build_obj.phase
+  status = {}
+  error = None
+
   try:
     status = build_logs.get_status(build_obj.uuid)
-  except BuildStatusRetrievalError:
-    status = {}
+  except BuildStatusRetrievalError as bsre:
     phase = 'cannot_load'
+    if SuperUserPermission().can():
+      error = str(bsre)
+    else:
+      error = 'Redis may be down. Please contact support.'
 
-  # If the status contains a heartbeat, then check to see if has been written in the last few
-  # minutes. If not, then the build timed out.
-  if phase != database.BUILD_PHASE.COMPLETE and phase != database.BUILD_PHASE.ERROR:
-    if status is not None and 'heartbeat' in status and status['heartbeat']:
-      heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat'])
-      if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1):
-        phase = database.BUILD_PHASE.INTERNAL_ERROR
+  if phase != 'cannot_load':
+    # If the status contains a heartbeat, then check to see if has been written in the last few
+    # minutes. If not, then the build timed out.
+    if phase != database.BUILD_PHASE.COMPLETE and phase != database.BUILD_PHASE.ERROR:
+      if status is not None and 'heartbeat' in status and status['heartbeat']:
+        heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat'])
+        if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1):
+          phase = database.BUILD_PHASE.INTERNAL_ERROR
 
-  # If the phase is internal error, return 'error' instead if the number of retries
-  # on the queue item is 0.
-  if phase == database.BUILD_PHASE.INTERNAL_ERROR:
-    retry = build_obj.queue_id and dockerfile_build_queue.has_retries_remaining(build_obj.queue_id)
-    if not retry:
-      phase = database.BUILD_PHASE.ERROR
+    # If the phase is internal error, return 'error' instead if the number of retries
+    # on the queue item is 0.
+    if phase == database.BUILD_PHASE.INTERNAL_ERROR:
+      retry = build_obj.queue_id and dockerfile_build_queue.has_retries_remaining(build_obj.queue_id)
+      if not retry:
+        phase = database.BUILD_PHASE.ERROR
 
   repo_namespace = build_obj.repository.namespace_user.username
   repo_name = build_obj.repository.name
@@ -134,7 +142,8 @@ def build_status_view(build_obj):
     'repository': {
       'namespace': repo_namespace,
       'name': repo_name
-    }
+    },
+    'error': error,
   }
 
   if can_write:

static/js/directives/ui/build-message.js

@@ -15,7 +15,7 @@ angular.module('quay').directive('buildMessage', function () {
       $scope.getBuildMessage = function (phase) {
         switch (phase) {
           case 'cannot_load':
-            return 'Cannot load build status - Please report this error';
+            return 'Cannot load build status';
 
           case 'starting':
           case 'initializing':

static/partials/build-view.html

@@ -19,8 +19,14 @@
         <!-- Build Information -->
         <div class="build-info-bar" build="build" show-time="false"></div>
 
+        <!-- Error Status -->
+        <div class="co-alert co-alert-warning" ng-show="originalBuild.error"
+             style="margin-bottom: 8px; margin-top: 12px;">
+          {{ originalBuild.error }}
+        </div>
+
         <!-- Current Status -->
-        <div class="build-status-header">
+        <div class="build-status-header" ng-show="!originalBuild.error">
           <span class="build-icon-message" ng-class="build.phase">
             <span class="cor-loader-inline" ng-if="isBuilding(build)"></span>
             <span ng-if="!isBuilding(build)">
@@ -57,7 +63,8 @@
         <div class="build-logs-view"
              build="originalBuild"
              use-timestamps="showLogTimestamps"
-             build-updated="setUpdatedBuild(build)"></div>
+             build-updated="setUpdatedBuild(build)"
+             ng-show="!originalBuild.error"></div>
       </div>
     </div>
   </div>