client: implement exponential backoff on temporary network errors

Uses a fuzzy timer to reduce chance of multiple clients synchronizing.

omaha/client/error.go

@@ -18,14 +18,40 @@ import (
 	"encoding/xml"
 	"errors"
 	"io"
+	"net"
 	"net/http"
+	"time"
 )
 
 var (
 	bodySizeError  = errors.New("http response exceeded 1MB")
 	bodyEmptyError = errors.New("http response was empty")
+
+	// default parameters for expNetBackoff
+	backoffStart = time.Second
+	backoffTries = 7
 )
 
+// retries and exponentially backs off for temporary network errors
+func expNetBackoff(f func() error) error {
+	var (
+		backoff = backoffStart
+		tries   = backoffTries
+	)
+	for {
+		err := f()
+		tries--
+		if tries <= 0 {
+			return err
+		}
+		if neterr, ok := err.(net.Error); !ok || !neterr.Temporary() {
+			return err
+		}
+		FuzzySleep(backoff, backoff)
+		backoff *= 2
+	}
+}
+
 // xml doesn't return the standard io.ErrUnexpectedEOF so check for both.
 func isUnexpectedEOF(err error) bool {
 	if xerr, ok := err.(*xml.SyntaxError); ok {

omaha/client/error_test.go

@@ -0,0 +1,49 @@
+// Copyright 2017 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"testing"
+	"time"
+)
+
+func init() {
+	// use quicker backoff for testing
+	backoffStart = time.Millisecond
+	backoffTries = 3
+}
+
+type tmpErr struct{}
+
+func (e tmpErr) Error() string   { return "fake temporary error" }
+func (e tmpErr) Temporary() bool { return true }
+func (e tmpErr) Timeout() bool   { return false }
+
+func TestExpNetBackoff(t *testing.T) {
+	tries := 0
+	err := expNetBackoff(func() error {
+		tries++
+		if tries < 2 {
+			return tmpErr{}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if tries != 2 {
+		t.Errorf("unexpected # of tries: %d", tries)
+	}
+}

omaha/client/http.go

@@ -19,7 +19,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"time"
 
@@ -28,7 +27,6 @@ import (
 
 const (
 	defaultTimeout = 90 * time.Second
-	defaultTries   = 7
 )
 
 // httpClient extends the standard http.Client to support xml encoding
@@ -79,14 +77,10 @@ func (hc *httpClient) Omaha(url string, req *omaha.Request) (resp *omaha.Respons
 		return nil, fmt.Errorf("omaha: failed to encode request: %v", err)
 	}
 
-	for i := 0; i < defaultTries; i++ {
+	expNetBackoff(func() error {
 		resp, err = hc.doPost(url, buf.Bytes())
-		if neterr, ok := err.(net.Error); ok && neterr.Temporary() {
-			// TODO(marineam): add exponential backoff
-			continue
-		}
-		break
-	}
+		return err
+	})
 	if err != nil {
 		return nil, fmt.Errorf("omaha: request failed: %v", err)
 	}