From 04df2410ec9fa5a497c3d94d25a3cf32c65cb75c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 31 May 2016 15:24:36 -0400 Subject: [PATCH] Add better errors if Redis is down Fixes #1497 --- data/buildlogs.py | 8 ++-- endpoints/api/build.py | 47 ++++++++++++++---------- static/js/directives/ui/build-message.js | 2 +- static/partials/build-view.html | 11 +++++- 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/data/buildlogs.py b/data/buildlogs.py index 07127b65f..f0101fd6e 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -61,8 +61,8 @@ class RedisBuildLogs(object): llen = self._redis.llen(self._logs_key(build_id)) log_entries = self._redis.lrange(self._logs_key(build_id), start_index, -1) return (llen, (json.loads(entry) for entry in log_entries)) - except redis.ConnectionError: - raise BuildStatusRetrievalError('Cannot retrieve build logs') + except redis.ConnectionError as ce: + raise BuildStatusRetrievalError('Cannot retrieve build logs: %s' % ce) def expire_log_entries(self, build_id): """ @@ -87,8 +87,8 @@ class RedisBuildLogs(object): """ try: fetched = self._redis.get(self._status_key(build_id)) - except redis.ConnectionError: - raise BuildStatusRetrievalError('Cannot retrieve build status') + except redis.ConnectionError as ce: + raise BuildStatusRetrievalError('Cannot retrieve build status: %s' % ce) return json.loads(fetched) if fetched else None diff --git a/endpoints/api/build.py b/endpoints/api/build.py index 972255955..133ad3616 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -8,11 +8,11 @@ import hashlib from flask import request from rfc3987 import parse as uri_parse -from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue +from app import userfiles as user_files, build_logs, log_archive, dockerfile_build_queue from buildtrigger.basehandler import BuildTriggerHandler from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, - ApiResource, internal_only, format_date, api, path_param, + ApiResource, internal_only, format_date, api, path_param, require_repo_admin) from endpoints.exception import Unauthorized, NotFound, InvalidRequest from endpoints.building import start_build, PreparedBuild @@ -20,7 +20,8 @@ from data import database from data import model from auth.auth_context import get_authenticated_user from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, - AdministerRepositoryPermission, AdministerOrganizationPermission) + AdministerRepositoryPermission, AdministerOrganizationPermission, + SuperUserPermission) from data.buildlogs import BuildStatusRetrievalError from util.names import parse_robot_username @@ -87,26 +88,33 @@ def trigger_view(trigger, can_read=False, can_admin=False, for_build=False): def build_status_view(build_obj): phase = build_obj.phase + status = {} + error = None + try: status = build_logs.get_status(build_obj.uuid) - except BuildStatusRetrievalError: - status = {} + except BuildStatusRetrievalError as bsre: phase = 'cannot_load' + if SuperUserPermission().can(): + error = str(bsre) + else: + error = 'Redis may be down. Please contact support.' - # If the status contains a heartbeat, then check to see if has been written in the last few - # minutes. If not, then the build timed out. - if phase != database.BUILD_PHASE.COMPLETE and phase != database.BUILD_PHASE.ERROR: - if status is not None and 'heartbeat' in status and status['heartbeat']: - heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat']) - if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1): - phase = database.BUILD_PHASE.INTERNAL_ERROR + if phase != 'cannot_load': + # If the status contains a heartbeat, then check to see if has been written in the last few + # minutes. If not, then the build timed out. + if phase != database.BUILD_PHASE.COMPLETE and phase != database.BUILD_PHASE.ERROR: + if status is not None and 'heartbeat' in status and status['heartbeat']: + heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat']) + if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1): + phase = database.BUILD_PHASE.INTERNAL_ERROR - # If the phase is internal error, return 'error' instead if the number of retries - # on the queue item is 0. - if phase == database.BUILD_PHASE.INTERNAL_ERROR: - retry = build_obj.queue_id and dockerfile_build_queue.has_retries_remaining(build_obj.queue_id) - if not retry: - phase = database.BUILD_PHASE.ERROR + # If the phase is internal error, return 'error' instead if the number of retries + # on the queue item is 0. + if phase == database.BUILD_PHASE.INTERNAL_ERROR: + retry = build_obj.queue_id and dockerfile_build_queue.has_retries_remaining(build_obj.queue_id) + if not retry: + phase = database.BUILD_PHASE.ERROR repo_namespace = build_obj.repository.namespace_user.username repo_name = build_obj.repository.name @@ -134,7 +142,8 @@ def build_status_view(build_obj): 'repository': { 'namespace': repo_namespace, 'name': repo_name - } + }, + 'error': error, } if can_write: diff --git a/static/js/directives/ui/build-message.js b/static/js/directives/ui/build-message.js index e0ac208d8..bf097d690 100644 --- a/static/js/directives/ui/build-message.js +++ b/static/js/directives/ui/build-message.js @@ -15,7 +15,7 @@ angular.module('quay').directive('buildMessage', function () { $scope.getBuildMessage = function (phase) { switch (phase) { case 'cannot_load': - return 'Cannot load build status - Please report this error'; + return 'Cannot load build status'; case 'starting': case 'initializing': diff --git a/static/partials/build-view.html b/static/partials/build-view.html index 43f48a7ce..d7e934936 100644 --- a/static/partials/build-view.html +++ b/static/partials/build-view.html @@ -19,8 +19,14 @@
+ +
+ {{ originalBuild.error }} +
+ -
+
@@ -57,7 +63,8 @@
+ build-updated="setUpdatedBuild(build)" + ng-show="!originalBuild.error">