vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Skip the database check filtering if not present, such as on warnings

Browse source
This commit is contained in:
Joseph Schorr 2019-02-04 16:36:02 -05:00
parent 18e5550f74
commit fcfc81806c
1 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
health/healthcheck.py
View file

@ -120,6 +120,8 @@ class LocalHealthCheck(HealthCheck):
class RDSAwareHealthCheck(HealthCheck): class RDSAwareHealthCheck(HealthCheck):
def __init__(self, app, config_provider, instance_keys, access_key, secret_key, def __init__(self, app, config_provider, instance_keys, access_key, secret_key,
db_instance='quay', region='us-east-1'): db_instance='quay', region='us-east-1'):
# Note: We skip the redis check because if redis is down, we don't want ELB taking the
# machines out of service. Redis is not considered a high avaliability-required service.
super(RDSAwareHealthCheck, self).__init__(app, config_provider, instance_keys, [ super(RDSAwareHealthCheck, self).__init__(app, config_provider, instance_keys, [
'redis', 'storage']) 'redis', 'storage'])
@ -133,23 +135,22 @@ class RDSAwareHealthCheck(HealthCheck):
return ['RDSAwareHealthCheck', 'ProductionHealthCheck'] return ['RDSAwareHealthCheck', 'ProductionHealthCheck']
def get_instance_health(self, service_statuses): def get_instance_health(self, service_statuses):
# Note: We skip the redis check because if redis is down, we don't want ELB taking the
# machines out of service. Redis is not considered a high avaliability-required service.
skip = [] skip = []
notes = [] notes = []
# If the database is marked as unhealthy, check the status of RDS directly. If RDS is # If the database is marked as unhealthy, check the status of RDS directly. If RDS is
# reporting as available, then the problem is with this instance. Otherwise, the problem is # reporting as available, then the problem is with this instance. Otherwise, the problem is
# with RDS, and so we skip the DB status so we can keep this machine as 'healthy'. # with RDS, and so we skip the DB status so we can keep this machine as 'healthy'.
db_healthy = service_statuses['database'] if 'database' in service_statuses:
if not db_healthy: db_healthy = service_statuses['database']
rds_status = self._get_rds_status() if not db_healthy:
notes.append('DB reports unhealthy; RDS status: %s' % rds_status) rds_status = self._get_rds_status()
notes.append('DB reports unhealthy; RDS status: %s' % rds_status)
# If the RDS is in any state but available, then we skip the DB check since it will # If the RDS is in any state but available, then we skip the DB check since it will
# fail and bring down the instance. # fail and bring down the instance.
if rds_status != 'available': if rds_status != 'available':
skip.append('database') skip.append('database')
return self.calculate_overall_health(service_statuses, skip=skip, notes=notes) return self.calculate_overall_health(service_statuses, skip=skip, notes=notes)