From d40844181a2522adf98ccf6e7b39341ac2685bed Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 2 Jun 2017 14:14:24 -0700 Subject: [PATCH 1/5] client: add fuzzy timer For randomizing update check intervals and backoff delays to reduce chance of DoSing the server if lots of clients start together. --- omaha/client/fuzzytime.go | 53 ++++++++++++++++++++++++++++++++++ omaha/client/fuzzytime_test.go | 32 ++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 omaha/client/fuzzytime.go create mode 100644 omaha/client/fuzzytime_test.go diff --git a/omaha/client/fuzzytime.go b/omaha/client/fuzzytime.go new file mode 100644 index 0000000..ca9669f --- /dev/null +++ b/omaha/client/fuzzytime.go @@ -0,0 +1,53 @@ +// Copyright 2017 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "math/rand" + "time" +) + +func init() { + // Ensure seeding the prng is never forgotten, that would defeat + // the whole point of using fuzzy timers to guard against a DoS. + rand.Seed(time.Now().UnixNano()) +} + +// FuzzyDuration randomizes the duration d within the range specified +// by fuzz. Specifically the value range is: [d-(fuzz/2), d+(fuzz/2)] +// The result will never be negative. +func FuzzyDuration(d, fuzz time.Duration) time.Duration { + if fuzz < 0 { + return d + } + // apply range [-fuzz/2, fuzz/2] + d += time.Duration(rand.Int63n(int64(fuzz)+1) - (int64(fuzz) / 2)) + if d < 0 { + return 0 + } + return d +} + +// FuzzyAfter waits for the fuzzy duration to elapse and then sends the +// current time on the returned channel. See FuzzyDuration. +func FuzzyAfter(d, fuzz time.Duration) <-chan time.Time { + return time.After(FuzzyDuration(d, fuzz)) +} + +// FuzzySleep pauses the current goroutine for the fuzzy duration d. +// See FuzzyDuration. +func FuzzySleep(d, fuzz time.Duration) { + time.Sleep(FuzzyDuration(d, fuzz)) +} diff --git a/omaha/client/fuzzytime_test.go b/omaha/client/fuzzytime_test.go new file mode 100644 index 0000000..1a6631d --- /dev/null +++ b/omaha/client/fuzzytime_test.go @@ -0,0 +1,32 @@ +// Copyright 2017 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "testing" + "time" +) + +func TestFuzzyDuration(t *testing.T) { + const d = time.Minute + for i := 0; i < 1000; i++ { + f := FuzzyDuration(d, d) + if f < d/2 { + t.Errorf("%d < %d", f, d/2) + } else if f > d+d/2 { + t.Errorf("%d > %d", f, d+d/2) + } + } +} From c9e5a6a602a6d8b329161a2a14c812325cc59abe Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 2 Jun 2017 17:10:27 -0700 Subject: [PATCH 2/5] client: define default version as a constant --- omaha/client/client.go | 6 +++++- omaha/client/machine_linux.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/omaha/client/client.go b/omaha/client/client.go index 3be8c44..b1b8a16 100644 --- a/omaha/client/client.go +++ b/omaha/client/client.go @@ -25,6 +25,10 @@ import ( "github.com/coreos/go-omaha/omaha" ) +const ( + defaultClientVersion = "go-omaha" +) + // Client supports managing multiple apps using a single server. type Client struct { apiClient *httpClient @@ -53,7 +57,7 @@ func New(serverURL, userID string) (*Client, error) { c := &Client{ apiClient: newHTTPClient(), - clientVersion: "go-omaha", + clientVersion: defaultClientVersion, userID: userID, sessionID: uuid.NewV4().String(), apps: make(map[string]*AppClient), diff --git a/omaha/client/machine_linux.go b/omaha/client/machine_linux.go index 3661318..a666d99 100644 --- a/omaha/client/machine_linux.go +++ b/omaha/client/machine_linux.go @@ -60,7 +60,7 @@ func NewMachineClient(serverURL string) (*Client, error) { c := &Client{ apiClient: newHTTPClient(), - clientVersion: "go-omaha", + clientVersion: defaultClientVersion, userID: string(machineID), sessionID: string(bootID), isMachine: true, From c88c5916bb202739e86dea80714ad7b569f2bae0 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 2 Jun 2017 18:33:30 -0700 Subject: [PATCH 3/5] client: implement exponential backoff on temporary network errors Uses a fuzzy timer to reduce chance of multiple clients synchronizing. --- omaha/client/error.go | 26 ++++++++++++++++++++ omaha/client/error_test.go | 49 ++++++++++++++++++++++++++++++++++++++ omaha/client/http.go | 12 +++------- 3 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 omaha/client/error_test.go diff --git a/omaha/client/error.go b/omaha/client/error.go index 2132295..9e1a80e 100644 --- a/omaha/client/error.go +++ b/omaha/client/error.go @@ -18,14 +18,40 @@ import ( "encoding/xml" "errors" "io" + "net" "net/http" + "time" ) var ( bodySizeError = errors.New("http response exceeded 1MB") bodyEmptyError = errors.New("http response was empty") + + // default parameters for expNetBackoff + backoffStart = time.Second + backoffTries = 7 ) +// retries and exponentially backs off for temporary network errors +func expNetBackoff(f func() error) error { + var ( + backoff = backoffStart + tries = backoffTries + ) + for { + err := f() + tries-- + if tries <= 0 { + return err + } + if neterr, ok := err.(net.Error); !ok || !neterr.Temporary() { + return err + } + FuzzySleep(backoff, backoff) + backoff *= 2 + } +} + // xml doesn't return the standard io.ErrUnexpectedEOF so check for both. func isUnexpectedEOF(err error) bool { if xerr, ok := err.(*xml.SyntaxError); ok { diff --git a/omaha/client/error_test.go b/omaha/client/error_test.go new file mode 100644 index 0000000..4b7bd6a --- /dev/null +++ b/omaha/client/error_test.go @@ -0,0 +1,49 @@ +// Copyright 2017 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "testing" + "time" +) + +func init() { + // use quicker backoff for testing + backoffStart = time.Millisecond + backoffTries = 3 +} + +type tmpErr struct{} + +func (e tmpErr) Error() string { return "fake temporary error" } +func (e tmpErr) Temporary() bool { return true } +func (e tmpErr) Timeout() bool { return false } + +func TestExpNetBackoff(t *testing.T) { + tries := 0 + err := expNetBackoff(func() error { + tries++ + if tries < 2 { + return tmpErr{} + } + return nil + }) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if tries != 2 { + t.Errorf("unexpected # of tries: %d", tries) + } +} diff --git a/omaha/client/http.go b/omaha/client/http.go index 2db32f4..13e4a7b 100644 --- a/omaha/client/http.go +++ b/omaha/client/http.go @@ -19,7 +19,6 @@ import ( "encoding/xml" "fmt" "io" - "net" "net/http" "time" @@ -28,7 +27,6 @@ import ( const ( defaultTimeout = 90 * time.Second - defaultTries = 7 ) // httpClient extends the standard http.Client to support xml encoding @@ -79,14 +77,10 @@ func (hc *httpClient) Omaha(url string, req *omaha.Request) (resp *omaha.Respons return nil, fmt.Errorf("omaha: failed to encode request: %v", err) } - for i := 0; i < defaultTries; i++ { + expNetBackoff(func() error { resp, err = hc.doPost(url, buf.Bytes()) - if neterr, ok := err.(net.Error); ok && neterr.Temporary() { - // TODO(marineam): add exponential backoff - continue - } - break - } + return err + }) if err != nil { return nil, fmt.Errorf("omaha: request failed: %v", err) } From 1b026dfef55354ded96b78e4ec98d8ed8f20cf49 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 2 Jun 2017 18:38:56 -0700 Subject: [PATCH 4/5] client: add fuzzy timer for update check and ping interval Uses the same timing parameters as update_engine. --- omaha/client/client.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/omaha/client/client.go b/omaha/client/client.go index b1b8a16..1cbf163 100644 --- a/omaha/client/client.go +++ b/omaha/client/client.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/url" + "time" "github.com/satori/go.uuid" @@ -27,6 +28,11 @@ import ( const ( defaultClientVersion = "go-omaha" + + // periodic update check and ping intervals + pingFuzz = 10 * time.Minute + pingDelay = 7 * time.Minute // first check after 2-12 minutes + pingInterval = 45 * time.Minute // check in every 40-50 minutes ) // Client supports managing multiple apps using a single server. @@ -37,6 +43,7 @@ type Client struct { userID string sessionID string isMachine bool + sentPing bool apps map[string]*AppClient } @@ -97,6 +104,16 @@ func (c *Client) SetClientVersion(clientVersion string) { c.clientVersion = clientVersion } +// NextPing returns a timer channel that will fire when the next update +// check or ping should be sent. +func (c *Client) NextPing() <-chan time.Time { + d := pingDelay + if c.sentPing { + d = pingInterval + } + return FuzzyAfter(d, pingFuzz) +} + // AppClient gets the application client for the given application ID. func (c *Client) AppClient(appID string) (*AppClient, error) { if app, ok := c.apps[appID]; ok { @@ -167,6 +184,8 @@ func (ac *AppClient) UpdateCheck() (*omaha.UpdateResponse, error) { app.AddPing() app.AddUpdateCheck() + ac.sentPing = true + appResp, err := ac.doReq(ac.apiEndpoint, req) if err != nil { return nil, err @@ -196,6 +215,8 @@ func (ac *AppClient) Ping() error { app := req.Apps[0] app.AddPing() + ac.sentPing = true + appResp, err := ac.doReq(ac.apiEndpoint, req) if err != nil { return err From bd1ae5648ee47bda13a6c073b3f0229185cae22d Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Mon, 5 Jun 2017 13:20:08 -0700 Subject: [PATCH 5/5] client: add rough example for using the client --- omaha/client/example_test.go | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 omaha/client/example_test.go diff --git a/omaha/client/example_test.go b/omaha/client/example_test.go new file mode 100644 index 0000000..8a7a076 --- /dev/null +++ b/omaha/client/example_test.go @@ -0,0 +1,100 @@ +// Copyright 2017 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "fmt" + "os" + //"os/signal" + "syscall" + + "github.com/coreos/go-omaha/omaha" +) + +func Example() { + // Launch a dummy server for our client to talk to. + s, err := omaha.NewTrivialServer("127.0.0.1:0") + if err != nil { + fmt.Println(err) + return + } + defer s.Destroy() + go s.Serve() + + // Configure our client. userID should be random but preserved + // across restarts. version is the current version of our app. + var ( + serverURL = "http://" + s.Addr().String() + userID = "8b10fc6d-30ca-49b2-b1a2-8185f03d522b" + appID = "5ca607f8-61b5-4692-90ce-30380ba05a98" + version = "1.0.0" + ) + c, err := NewAppClient(serverURL, userID, appID, version) + if err != nil { + fmt.Println(err) + return + } + + // Client version is the name and version of this updater. + c.SetClientVersion("example-0.0.1") + + // Use SIGUSR1 to trigger immediate update checks. + sigc := make(chan os.Signal, 1) + //signal.Notify(sigc, syscall.SIGUSR1) + sigc <- syscall.SIGUSR1 // Fake it + + //for { + var source string + select { + case <-sigc: + source = "ondemandupdate" + case <-c.NextPing(): + source = "scheduler" + } + + // TODO: pass source to UpdateCheck + _ = source + // If updates are disabled call c.Ping() instead. + update, err := c.UpdateCheck() + if err != nil { + fmt.Println(err) + //continue + return + } + + // Download new application version. + c.Event(&omaha.EventRequest{ + Type: omaha.EventTypeUpdateDownloadFinished, + Result: omaha.EventResultSuccess, + }) + + // Install new application version here. + c.Event(&omaha.EventRequest{ + Type: omaha.EventTypeUpdateComplete, + Result: omaha.EventResultSuccess, + }) + + // Restart, new application is now running. + c.SetVersion(update.Manifest.Version) + c.Event(&omaha.EventRequest{ + Type: omaha.EventTypeUpdateComplete, + Result: omaha.EventResultSuccessReboot, + }) + + //} + + // Output: + // omaha: update status noupdate +}