go services: do not fail (+retry) profiler init

Signed-off-by: Ahmet Alp Balkan <ahmetb@google.com>

src/checkoutservice/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"time"
 
+	"cloud.google.com/go/profiler"
 	"github.com/google/uuid"
 	"go.opencensus.io/exporter/stackdriver"
 	"go.opencensus.io/plugin/ocgrpc"
@@ -18,8 +19,6 @@ import (
 
 	pb "checkoutservice/genproto"
 	money "checkoutservice/money"
-
-	"cloud.google.com/go/profiler"
 )
 
 const (
@@ -37,22 +36,14 @@ type checkoutService struct {
 }
 
 func main() {
-	if err := profiler.Start(profiler.Config{
+	go initTracing()
-		Service:        "cartservice",
+	go initProfiling("checkoutservice", "1.0.0")
-		ServiceVersion: "1.0.0",
-		// ProjectID must be set if not running on GCP.
-		// ProjectID: "my-project",
-	}); err != nil {
-		log.Fatalf("failed to start profiler: %+v", err)
-	}
 
 	port := listenPort
 	if os.Getenv("PORT") != "" {
 		port = os.Getenv("PORT")
 	}
 
-	go initTracing()
-
 	svc := new(checkoutService)
 	mustMapEnv(&svc.shippingSvcAddr, "SHIPPING_SERVICE_ADDR")
 	mustMapEnv(&svc.productCatalogSvcAddr, "PRODUCT_CATALOG_SERVICE_ADDR")
@@ -93,6 +84,25 @@ func initTracing() {
 	log.Printf("warning: could not initialize stackdriver exporter after retrying, giving up")
 }
 
+func initProfiling(service, version string) {
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		if err := profiler.Start(profiler.Config{
+			Service:        service,
+			ServiceVersion: version,
+			// ProjectID must be set if not running on GCP.
+			// ProjectID: "my-project",
+		}); err != nil {
+			log.Printf("warn: failed to start profiler: %+v", err)
+		}
+		d := time.Second * 10 * time.Duration(i)
+		log.Printf("sleeping %v to retry initializing stackdriver profiler", d)
+		time.Sleep(d)
+	}
+	log.Printf("warning: could not initialize stackdriver profiler after retrying, giving up")
+}
+
 func mustMapEnv(target *string, envKey string) {
 	v := os.Getenv(envKey)
 	if v == "" {

src/frontend/main.go

@@ -63,22 +63,14 @@ type frontendServer struct {
 }
 
 func main() {
-	if err := profiler.Start(profiler.Config{
+	go initProfiling(frontend, "1.0.0")
-		Service:        "frontendservice",
+	go initTracing(log)
-		ServiceVersion: "1.0.0",
-		// ProjectID must be set if not running on GCP.
-		// ProjectID: "my-project",
-	}); err != nil {
-		log.Fatalf("failed to start profiler: %+v", err)
-	}
 
 	ctx := context.Background()
 	log := logrus.New()
 	log.Level = logrus.DebugLevel
 	log.Formatter = &logrus.TextFormatter{}
 
-	go initTracing(log)
-
 	srvPort := port
 	if os.Getenv("PORT") != "" {
 		srvPort = os.Getenv("PORT")
@@ -142,6 +134,25 @@ func initTracing(log logrus.FieldLogger) {
 	log.Warn("could not initialize stackdriver exporter after retrying, giving up")
 }
 
+func initProfiling(service, version string) {
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		if err := profiler.Start(profiler.Config{
+			Service:        service,
+			ServiceVersion: version,
+			// ProjectID must be set if not running on GCP.
+			// ProjectID: "my-project",
+		}); err != nil {
+			log.Printf("warn: failed to start profiler: %+v", err)
+		}
+		d := time.Second * 10 * time.Duration(i)
+		log.Printf("sleeping %v to retry initializing stackdriver profiler", d)
+		time.Sleep(d)
+	}
+	log.Printf("warning: could not initialize stackdriver profiler after retrying, giving up")
+}
+
 func mustMapEnv(target *string, envKey string) {
 	v := os.Getenv(envKey)
 	if v == "" {

src/productcatalogservice/server.go

@@ -88,18 +88,9 @@ var catalog = []*pb.Product{
 }
 
 func main() {
-	if err := profiler.Start(profiler.Config{
-		Service:        "productcatalogservice",
-		ServiceVersion: "1.0.0",
-		// ProjectID must be set if not running on GCP.
-		// ProjectID: "my-project",
-	}); err != nil {
-		log.Fatalf("failed to start profiler: %+v", err)
-	}
-
-	flag.Parse()
-
 	go initTracing()
+	go initProfiling("productcatalogservice", "1.0.0")
+	flag.Parse()
 
 	log.Printf("starting grpc server at :%d", *port)
 	run(*port)
@@ -137,6 +128,25 @@ func initTracing() {
 	log.Printf("warning: could not initialize stackdriver exporter after retrying, giving up")
 }
 
+func initProfiling(service, version string) {
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		if err := profiler.Start(profiler.Config{
+			Service:        service,
+			ServiceVersion: version,
+			// ProjectID must be set if not running on GCP.
+			// ProjectID: "my-project",
+		}); err != nil {
+			log.Printf("warn: failed to start profiler: %+v", err)
+		}
+		d := time.Second * 10 * time.Duration(i)
+		log.Printf("sleeping %v to retry initializing stackdriver profiler", d)
+		time.Sleep(d)
+	}
+	log.Printf("warning: could not initialize stackdriver profiler after retrying, giving up")
+}
+
 type productCatalog struct{}
 
 func (p *productCatalog) ListProducts(context.Context, *pb.Empty) (*pb.ListProductsResponse, error) {

src/shippingservice/main.go

@@ -23,6 +23,9 @@ const (
 )
 
 func main() {
+	go initTracing()
+	go initProfiling("shippingservice", "1.0.0")
+
 	port := defaultPort
 	if value, ok := os.LookupEnv("APP_PORT"); ok {
 		port = value
@@ -38,8 +41,6 @@ func main() {
 		log.Fatalf("failed to start profiler: %+v", err)
 	}
 
-	go initTracing()
-
 	lis, err := net.Listen("tcp", port)
 	if err != nil {
 		log.Fatalf("failed to listen: %v", err)
@@ -116,3 +117,22 @@ func initTracing() {
 	}
 	log.Printf("warning: could not initialize stackdriver exporter after retrying, giving up")
 }
+
+func initProfiling(service, version string) {
+	// TODO(ahmetb) this method is duplicated in other microservices using Go
+	// since they are not sharing packages.
+	for i := 1; i <= 3; i++ {
+		if err := profiler.Start(profiler.Config{
+			Service:        service,
+			ServiceVersion: version,
+			// ProjectID must be set if not running on GCP.
+			// ProjectID: "my-project",
+		}); err != nil {
+			log.Printf("warn: failed to start profiler: %+v", err)
+		}
+		d := time.Second * 10 * time.Duration(i)
+		log.Printf("sleeping %v to retry initializing stackdriver profiler", d)
+		time.Sleep(d)
+	}
+	log.Printf("warning: could not initialize stackdriver profiler after retrying, giving up")
+}