diff --git a/plugins/client.go b/plugins/client.go index 00ca105..d531fa4 100644 --- a/plugins/client.go +++ b/plugins/client.go @@ -31,6 +31,10 @@ type Client struct { } func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) error { + return c.callWithRetry(serviceMethod, args, ret, true) +} + +func (c *Client) callWithRetry(serviceMethod string, args interface{}, ret interface{}, retry bool) error { var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(args); err != nil { return err @@ -50,12 +54,16 @@ func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) e for { resp, err := c.http.Do(req) if err != nil { + if !retry { + return err + } + timeOff := backoff(retries) - if timeOff+time.Since(start) > defaultTimeOut { + if abort(start, timeOff) { return err } retries++ - logrus.Warn("Unable to connect to plugin: %s, retrying in %ds\n", c.addr, timeOff) + logrus.Warnf("Unable to connect to plugin: %s, retrying in %v", c.addr, timeOff) time.Sleep(timeOff) continue } @@ -73,7 +81,7 @@ func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) e } func backoff(retries int) time.Duration { - b, max := float64(1), float64(defaultTimeOut) + b, max := 1, defaultTimeOut for b < max && retries > 0 { b *= 2 retries-- @@ -81,7 +89,11 @@ func backoff(retries int) time.Duration { if b > max { b = max } - return time.Duration(b) + return time.Duration(b) * time.Second +} + +func abort(start time.Time, timeOff time.Duration) bool { + return timeOff+time.Since(start) > time.Duration(defaultTimeOut)*time.Second } func configureTCPTransport(tr *http.Transport, proto, addr string) { diff --git a/plugins/client_test.go b/plugins/client_test.go index b414ecb..0f7cd34 100644 --- a/plugins/client_test.go +++ b/plugins/client_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "reflect" "testing" + "time" ) var ( @@ -27,7 +28,7 @@ func teardownRemotePluginServer() { func TestFailedConnection(t *testing.T) { c := NewClient("tcp://127.0.0.1:1") - err := c.Call("Service.Method", nil, nil) + err := c.callWithRetry("Service.Method", nil, nil, false) if err == nil { t.Fatal("Unexpected successful connection") } @@ -61,3 +62,44 @@ func TestEchoInputOutput(t *testing.T) { t.Fatalf("Expected %v, was %v\n", m, output) } } + +func TestBackoff(t *testing.T) { + cases := []struct { + retries int + expTimeOff time.Duration + }{ + {0, time.Duration(1)}, + {1, time.Duration(2)}, + {2, time.Duration(4)}, + {4, time.Duration(16)}, + {6, time.Duration(30)}, + {10, time.Duration(30)}, + } + + for _, c := range cases { + s := c.expTimeOff * time.Second + if d := backoff(c.retries); d != s { + t.Fatalf("Retry %v, expected %v, was %v\n", c.retries, s, d) + } + } +} + +func TestAbortRetry(t *testing.T) { + cases := []struct { + timeOff time.Duration + expAbort bool + }{ + {time.Duration(1), false}, + {time.Duration(2), false}, + {time.Duration(10), false}, + {time.Duration(30), true}, + {time.Duration(40), true}, + } + + for _, c := range cases { + s := c.timeOff * time.Second + if a := abort(time.Now(), s); a != c.expAbort { + t.Fatalf("Duration %v, expected %v, was %v\n", c.timeOff, s, a) + } + } +}