Have the health check also ping the registry endpoint to make sure it is functional.

endpoints/index.py

@@ -380,6 +380,11 @@ def get_search():
   resp.mimetype = 'application/json'
   return resp
 
+# Note: This is *not* part of the Docker index spec. This is here for our own health check,
+# since we have nginx handle the _ping below.
+@index.route('/_internal_ping')
+def internal_ping():
+  return make_response('true', 200)
 
 @index.route('/_ping')
 @index.route('/_ping')

endpoints/web.py

@@ -156,11 +156,21 @@ def v1():
 @web.route('/health', methods=['GET'])
 @no_cache
 def health():
+  client = app.config['HTTPCLIENT']
+
   db_healthy = model.check_health()
   buildlogs_healthy = build_logs.check_health()
 
+  hostname_parts = app.config['SERVER_HOSTNAME'].split(':')
+  port = ''
+  if len(hostname_parts) == 2:
+    port = ':' + hostname_parts[1]
+
+  registry_url = '%s://localhost%s/v1/_internal_ping' % (app.config['PREFERRED_URL_SCHEME'], port)
+  registry_healthy = client.get(registry_url, verify=False).status_code == 200
+
   check = HealthCheck.get_check(app.config['HEALTH_CHECKER'][0], app.config['HEALTH_CHECKER'][1])
-  (data, is_healthy) = check.conduct_healthcheck(db_healthy, buildlogs_healthy)
+  (data, is_healthy) = check.conduct_healthcheck(db_healthy, buildlogs_healthy, registry_healthy)
 
   response = jsonify(dict(data=data, is_healthy=is_healthy))
   response.status_code = 200 if is_healthy else 503

health/healthcheck.py

@@ -7,7 +7,7 @@ class HealthCheck(object):
   def __init__(self):
     pass
 
-  def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
+  def conduct_healthcheck(self, db_healthy, buildlogs_healthy, registry_healthy):
     """
     Conducts any custom healthcheck work, returning a dict representing the HealthCheck
     output and a boolean indicating whether the instance is healthy.
@@ -31,10 +31,11 @@ class LocalHealthCheck(HealthCheck):
   def check_name(cls):
     return 'LocalHealthCheck'
 
-  def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
+  def conduct_healthcheck(self, db_healthy, buildlogs_healthy, registry_healthy):
     data = {
       'db_healthy': db_healthy,
-      'buildlogs_healthy': buildlogs_healthy
+      'buildlogs_healthy': buildlogs_healthy,
+      'registry_healthy': registry_healthy
     }
 
     return (data, db_healthy and buildlogs_healthy)
@@ -49,10 +50,11 @@ class ProductionHealthCheck(HealthCheck):
   def check_name(cls):
     return 'ProductionHealthCheck'
 
-  def conduct_healthcheck(self, db_healthy, buildlogs_healthy):
+  def conduct_healthcheck(self, db_healthy, buildlogs_healthy, registry_healthy):
     data = {
       'db_healthy': db_healthy,
-      'buildlogs_healthy': buildlogs_healthy
+      'buildlogs_healthy': buildlogs_healthy,
+      'registry_healthy': registry_healthy
     }
 
     # Only report unhealthy if the machine cannot connect to the DB. Redis isn't required for
@@ -81,4 +83,4 @@ class ProductionHealthCheck(HealthCheck):
       # requests once RDS comes back up.
       return (data, not is_rds_working)
 
-    return (data, db_healthy)
+    return (data, db_healthy and registry_healthy)