Provide simple storage driver health check

To ensure the ensure the web application is properly operating, we've added a
periodic health check for the storage driver. If the health check fails three
times in a row, the registry will serve 503 response status for any request
until the condition is resolved. The condition is reported in the response body
and via the /debug/health endpoint.

To ensure that all drivers will properly operate with this health check, a
function has been added to the driver testsuite.

Signed-off-by: Stephen J Day <stephen.day@docker.com>

health/health_test.go

@@ -2,6 +2,7 @@ package health
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -45,3 +46,62 @@ func TestReturns503IfThereAreErrorChecks(t *testing.T) {
 		t.Errorf("Did not get a 503.")
 	}
 }
+
+// TestHealthHandler ensures that our handler implementation correct protects
+// the web application when things aren't so healthy.
+func TestHealthHandler(t *testing.T) {
+	// clear out existing checks.
+	registeredChecks = make(map[string]Checker)
+
+	// protect an http server
+	handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+
+	// wrap it in our health handler
+	handler = Handler(handler)
+
+	// use this swap check status
+	updater := NewStatusUpdater()
+	Register("test_check", updater)
+
+	// now, create a test server
+	server := httptest.NewServer(handler)
+
+	checkUp := func(t *testing.T, message string) {
+		resp, err := http.Get(server.URL)
+		if err != nil {
+			t.Fatalf("error getting success status: %v", err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusNoContent {
+			t.Fatalf("unexpected response code from server when %s: %d != %d", message, resp.StatusCode, http.StatusNoContent)
+		}
+		// NOTE(stevvooe): we really don't care about the body -- the format is
+		// not standardized or supported, yet.
+	}
+
+	checkDown := func(t *testing.T, message string) {
+		resp, err := http.Get(server.URL)
+		if err != nil {
+			t.Fatalf("error getting down status: %v", err)
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode != http.StatusServiceUnavailable {
+			t.Fatalf("unexpected response code from server when %s: %d != %d", message, resp.StatusCode, http.StatusServiceUnavailable)
+		}
+	}
+
+	// server should be up
+	checkUp(t, "initial health check")
+
+	// now, we fail the health check
+	updater.Update(fmt.Errorf("the server is now out of commission"))
+	checkDown(t, "server should be down") // should be down
+
+	// bring server back up
+	updater.Update(nil)
+	checkUp(t, "when server is back up") // now we should be back up.
+}