I hate Redis!

- Remove redis check from our health endpoint in prod entirely
- Have the redis check have a maximum timeout of 1 second

data/buildlogs.py

@@ -18,7 +18,11 @@ class RedisBuildLogs(object):
   PHASE = 'phase'
 
   def __init__(self, redis_config):
-    self._redis = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
+    args = dict(redis_config)
+    args.update({'socket_connect_timeout': 5})
+
+    self._redis_config = redis_config
+    self._redis = redis.StrictRedis(**args)
 
   @staticmethod
   def _logs_key(build_id):
@@ -94,12 +98,16 @@ class RedisBuildLogs(object):
 
   def check_health(self):
     try:
-      if not self._redis.ping() == True:
+      args = dict(self._redis_config)
+      args.update({'socket_connect_timeout': 1, 'socket_timeout': 1})
+
+      connection = redis.StrictRedis(**args)
+      if not connection.ping() == True:
         return False
 
       # Ensure we can write and read a key.
-      self._redis.set(self._health_key(), time.time())
+      connection.set(self._health_key(), time.time())
-      self._redis.get(self._health_key())
+      connection.get(self._health_key())
 
       return True
     except redis.ConnectionError:

health/healthcheck.py

@@ -10,16 +10,17 @@ def get_healthchecker(app, config_provider):
 
 
 class HealthCheck(object):
-  def __init__(self, app, config_provider):
+  def __init__(self, app, config_provider, instance_skips=None):
     self.app = app
     self.config_provider = config_provider
+    self.instance_skips = instance_skips or []
 
   def check_instance(self):
     """
     Conducts a check on this specific instance, returning a dict representing the HealthCheck
     output and a number indicating the health check response code.
     """
-    service_statuses = check_all_services(self.app)
+    service_statuses = check_all_services(self.app, skip=self.instance_skips)
     return self.get_instance_health(service_statuses)
 
   def check_endtoend(self):
@@ -80,7 +81,7 @@ class LocalHealthCheck(HealthCheck):
 
 class ProductionHealthCheck(HealthCheck):
   def __init__(self, app, config_provider, access_key, secret_key, db_instance='quay'):
-    super(ProductionHealthCheck, self).__init__(app, config_provider)
+    super(ProductionHealthCheck, self).__init__(app, config_provider, ['redis'])
     self.access_key = access_key
     self.secret_key = secret_key
     self.db_instance = db_instance
@@ -92,7 +93,7 @@ class ProductionHealthCheck(HealthCheck):
   def get_instance_health(self, service_statuses):
     # Note: We skip the redis check because if redis is down, we don't want ELB taking the
     # machines out of service. Redis is not considered a high avaliability-required service.
-    skip = ['redis']
+    skip = []
     notes = []
 
     # If the database is marked as unhealthy, check the status of RDS directly. If RDS is

health/services.py

@@ -39,10 +39,13 @@ _SERVICES = {
   'redis': _check_redis
 }
 
-def check_all_services(app):
+def check_all_services(app, skip):
   """ Returns a dictionary containing the status of all the services defined. """
   status = {}
   for name in _SERVICES:
+    if name in skip:
+      continue
+
     status[name] = _SERVICES[name](app)
 
-  return status
+  return status