diff --git a/data/buildlogs.py b/data/buildlogs.py index e9ea4a78f..ac58bd368 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -18,7 +18,11 @@ class RedisBuildLogs(object): PHASE = 'phase' def __init__(self, redis_config): - self._redis = redis.StrictRedis(socket_connect_timeout=5, **redis_config) + args = dict(redis_config) + args.update({'socket_connect_timeout': 5}) + + self._redis_config = redis_config + self._redis = redis.StrictRedis(**args) @staticmethod def _logs_key(build_id): @@ -94,12 +98,16 @@ class RedisBuildLogs(object): def check_health(self): try: - if not self._redis.ping() == True: + args = dict(self._redis_config) + args.update({'socket_connect_timeout': 1, 'socket_timeout': 1}) + + connection = redis.StrictRedis(**args) + if not connection.ping() == True: return False # Ensure we can write and read a key. - self._redis.set(self._health_key(), time.time()) - self._redis.get(self._health_key()) + connection.set(self._health_key(), time.time()) + connection.get(self._health_key()) return True except redis.ConnectionError: diff --git a/health/healthcheck.py b/health/healthcheck.py index c212c694d..c015adca2 100644 --- a/health/healthcheck.py +++ b/health/healthcheck.py @@ -10,16 +10,17 @@ def get_healthchecker(app, config_provider): class HealthCheck(object): - def __init__(self, app, config_provider): + def __init__(self, app, config_provider, instance_skips=None): self.app = app self.config_provider = config_provider + self.instance_skips = instance_skips or [] def check_instance(self): """ Conducts a check on this specific instance, returning a dict representing the HealthCheck output and a number indicating the health check response code. """ - service_statuses = check_all_services(self.app) + service_statuses = check_all_services(self.app, skip=self.instance_skips) return self.get_instance_health(service_statuses) def check_endtoend(self): @@ -80,7 +81,7 @@ class LocalHealthCheck(HealthCheck): class ProductionHealthCheck(HealthCheck): def __init__(self, app, config_provider, access_key, secret_key, db_instance='quay'): - super(ProductionHealthCheck, self).__init__(app, config_provider) + super(ProductionHealthCheck, self).__init__(app, config_provider, ['redis']) self.access_key = access_key self.secret_key = secret_key self.db_instance = db_instance @@ -92,7 +93,7 @@ class ProductionHealthCheck(HealthCheck): def get_instance_health(self, service_statuses): # Note: We skip the redis check because if redis is down, we don't want ELB taking the # machines out of service. Redis is not considered a high avaliability-required service. - skip = ['redis'] + skip = [] notes = [] # If the database is marked as unhealthy, check the status of RDS directly. If RDS is diff --git a/health/services.py b/health/services.py index ce6112651..bf108595a 100644 --- a/health/services.py +++ b/health/services.py @@ -39,10 +39,13 @@ _SERVICES = { 'redis': _check_redis } -def check_all_services(app): +def check_all_services(app, skip): """ Returns a dictionary containing the status of all the services defined. """ status = {} for name in _SERVICES: + if name in skip: + continue + status[name] = _SERVICES[name](app) - return status \ No newline at end of file + return status