Make the DB health check first attempt a simple DB connection. If the database is in the middle of a failover, this will fail after 3 seconds (the connection timeout specified), rather than hanging and causing the ELB health checks to timeout and fail.

data/model/legacy.py

@@ -14,7 +14,7 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor
                            ExternalNotificationEvent, ExternalNotificationMethod,
                            RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
                            DerivedImageStorage, ImageStorageTransformation, random_string_generator,
-                           db, BUILD_PHASE, QuayUserField)
+                           db, BUILD_PHASE, QuayUserField, validate_database_url)
 from peewee import JOIN_LEFT_OUTER, fn
 from util.validation import (validate_username, validate_email, validate_password,
                              INVALID_PASSWORD_MESSAGE)
@@ -2257,11 +2257,20 @@ def delete_user(user):
   # TODO: also delete any repository data associated
 
 
-def check_health():
+def check_health(app_config):
+  # Attempt to connect to the database first. If the DB is not responding,
+  # using the validate_database_url will timeout quickly, as opposed to
+  # making a normal connect which will just hang (thus breaking the health
+  # check).
+  try:
+    validate_database_url(app_config['DB_URI'], connect_timeout=3)
+  except Exception:
+    logger.exception('Could not connect to the database')
+    return False
+
   # We will connect to the db, check that it contains some log entry kinds
   try:
-    found_count = LogEntryKind.select().count()
-    return found_count > 0
+    return bool(list(LogEntryKind.select().limit(1)))
   except:
     return False