add retries for stackdriver exporters in go

Signed-off-by: Ahmet Alp Balkan <ahmetb@google.com>

src/checkoutservice/main.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"time"
 
 	"github.com/google/uuid"
 	"go.opencensus.io/exporter/stackdriver"
@@ -39,7 +40,7 @@ func main() {
 		port = os.Getenv("PORT")
 	}
 
-	initTracing()
+	go initTracing()
 
 	svc := new(checkoutService)
 	mustMapEnv(&svc.shippingSvcAddr, "SHIPPING_SERVICE_ADDR")
@@ -62,15 +63,23 @@ func main() {
 }
 
 func initTracing() {
-	exporter, err := stackdriver.NewExporter(stackdriver.Options{})
-	if err != nil {
-		log.Printf("failed to initialize stackdriver exporter: %+v", err)
-		log.Println("skipping uploading traces to stackdriver")
-	} else {
-		trace.RegisterExporter(exporter)
-		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
-		log.Println("registered stackdriver tracing")
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		exporter, err := stackdriver.NewExporter(stackdriver.Options{})
+		if err != nil {
+			log.Printf("info: failed to initialize stackdriver exporter: %+v", err)
+		} else {
+			trace.RegisterExporter(exporter)
+			trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+			log.Print("registered stackdriver tracing")
+			return
+		}
+		d := time.Second * 10 * time.Duration(i)
+		log.Printf("sleeping %v to retry initializing stackdriver exporter", d)
+		time.Sleep(d)
 	}
+	log.Printf("warning: could not initialize stackdriver exporter after retrying, giving up")
 }
 
 func mustMapEnv(target *string, envKey string) {

src/frontend/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"log"
 	"net/http"
 	"os"
 	"time"
@@ -67,7 +66,7 @@ func main() {
 	log.Level = logrus.DebugLevel
 	log.Formatter = &logrus.TextFormatter{}
 
-	initTracing()
+	go initTracing(log)
 
 	srvPort := port
 	if os.Getenv("PORT") != "" {
@@ -111,16 +110,25 @@ func main() {
 	log.Fatal(http.ListenAndServe(addr+":"+srvPort, handler))
 }
 
-func initTracing() {
-	exporter, err := stackdriver.NewExporter(stackdriver.Options{})
-	if err != nil {
-		log.Printf("failed to initialize stackdriver exporter: %+v", err)
-		log.Println("skipping uploading traces to stackdriver")
-	} else {
-		trace.RegisterExporter(exporter)
-		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
-		log.Println("registered stackdriver tracing")
+func initTracing(log logrus.FieldLogger) {
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		log = log.WithField("retry", i)
+		exporter, err := stackdriver.NewExporter(stackdriver.Options{})
+		if err != nil {
+			log.Warnf("failed to initialize stackdriver exporter: %+v", err)
+		} else {
+			trace.RegisterExporter(exporter)
+			trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+			log.Info("registered stackdriver tracing")
+			return
+		}
+		d := time.Second * 20 * time.Duration(i)
+		log.Debugf("sleeping %v to retry initializing stackdriver exporter", d)
+		time.Sleep(d)
 	}
+	log.Warn("could not initialize stackdriver exporter after retrying, giving up")
 }
 
 func mustMapEnv(target *string, envKey string) {

src/productcatalogservice/server.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net"
 	"strings"
+	"time"
 
 	pb "./genproto"
 	"go.opencensus.io/exporter/stackdriver"
@@ -88,7 +89,7 @@ var catalog = []*pb.Product{
 func main() {
 	flag.Parse()
 
-	initTracing()
+	go initTracing()
 
 	log.Printf("starting grpc server at :%d", *port)
 	run(*port)
@@ -107,15 +108,23 @@ func run(port int) string {
 }
 
 func initTracing() {
-	exporter, err := stackdriver.NewExporter(stackdriver.Options{})
-	if err != nil {
-		log.Printf("failed to initialize stackdriver exporter: %+v", err)
-		log.Println("skipping uploading traces to stackdriver")
-	} else {
-		trace.RegisterExporter(exporter)
-		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
-		log.Println("registered stackdriver")
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		exporter, err := stackdriver.NewExporter(stackdriver.Options{})
+		if err != nil {
+			log.Printf("info: failed to initialize stackdriver exporter: %+v", err)
+		} else {
+			trace.RegisterExporter(exporter)
+			trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+			log.Print("registered stackdriver tracing")
+			return
+		}
+		d := time.Second * 10 * time.Duration(i)
+		log.Printf("sleeping %v to retry initializing stackdriver exporter", d)
+		time.Sleep(d)
 	}
+	log.Printf("warning: could not initialize stackdriver exporter after retrying, giving up")
 }
 
 type productCatalog struct{}

src/shippingservice/main.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"time"
 
 	"go.opencensus.io/exporter/stackdriver"
 	"go.opencensus.io/plugin/ocgrpc"
@@ -20,6 +21,30 @@ const (
 	defaultPort = "50051"
 )
 
+func main() {
+	port := defaultPort
+	if value, ok := os.LookupEnv("APP_PORT"); ok {
+		port = value
+	}
+	port = fmt.Sprintf(":%s", port)
+
+	go initTracing()
+
+	lis, err := net.Listen("tcp", port)
+	if err != nil {
+		log.Fatalf("failed to listen: %v", err)
+	}
+	s := grpc.NewServer(grpc.StatsHandler(&ocgrpc.ServerHandler{}))
+	pb.RegisterShippingServiceServer(s, &server{})
+	log.Printf("Shipping Service listening on port %s", port)
+
+	// Register reflection service on gRPC server.
+	reflection.Register(s)
+	if err := s.Serve(lis); err != nil {
+		log.Fatalf("failed to serve: %v", err)
+	}
+}
+
 // server controls RPC service responses.
 type server struct{}
 
@@ -62,38 +87,22 @@ func (s *server) ShipOrder(ctx context.Context, in *pb.ShipOrderRequest) (*pb.Sh
 	}, nil
 }
 
-func main() {
-	port := defaultPort
-	if value, ok := os.LookupEnv("APP_PORT"); ok {
-		port = value
-	}
-	port = fmt.Sprintf(":%s", port)
-
-	initTracing()
-
-	lis, err := net.Listen("tcp", port)
-	if err != nil {
-		log.Fatalf("failed to listen: %v", err)
-	}
-	s := grpc.NewServer(grpc.StatsHandler(&ocgrpc.ServerHandler{}))
-	pb.RegisterShippingServiceServer(s, &server{})
-	log.Printf("Shipping Service listening on port %s", port)
-
-	// Register reflection service on gRPC server.
-	reflection.Register(s)
-	if err := s.Serve(lis); err != nil {
-		log.Fatalf("failed to serve: %v", err)
-	}
-}
-
 func initTracing() {
-	exporter, err := stackdriver.NewExporter(stackdriver.Options{})
-	if err != nil {
-		log.Printf("failed to initialize stackdriver exporter: %+v", err)
-		log.Println("skipping uploading traces to stackdriver")
-	} else {
-		trace.RegisterExporter(exporter)
-		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
-		log.Println("registered stackdriver tracing")
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		exporter, err := stackdriver.NewExporter(stackdriver.Options{})
+		if err != nil {
+			log.Printf("info: failed to initialize stackdriver exporter: %+v", err)
+		} else {
+			trace.RegisterExporter(exporter)
+			trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+			log.Print("registered stackdriver tracing")
+			return
+		}
+		d := time.Second * 10 * time.Duration(i)
+		log.Printf("sleeping %v to retry initializing stackdriver exporter", d)
+		time.Sleep(d)
 	}
+	log.Printf("warning: could not initialize stackdriver exporter after retrying, giving up")
 }