From 90e43a87442d7a9fb04e326f9a970ac9d7f5390b Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Wed, 2 Jan 2019 13:34:01 -0800 Subject: [PATCH 01/23] move img/ to docs/img (#110) Signed-off-by: Ahmet Alp Balkan --- README.md | 4 ++-- {img => docs/img}/architecture-diagram.png | Bin {img => docs/img}/hipster-shop-frontend-1.png | Bin {img => docs/img}/hipster-shop-frontend-2.png | Bin 4 files changed, 2 insertions(+), 2 deletions(-) rename {img => docs/img}/architecture-diagram.png (100%) rename {img => docs/img}/hipster-shop-frontend-1.png (100%) rename {img => docs/img}/hipster-shop-frontend-2.png (100%) diff --git a/README.md b/README.md index bc075b2..9829ab3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Stackdriver, gRPC** and similar cloud-native technologies nowadays. | Home Page | Checkout Screen | |-----------|-----------------| -| [![Screenshot of store homepage](./img/hipster-shop-frontend-1.png)](./img/hipster-shop-frontend-1.png) | [![Screenshot of checkout screen](./img/hipster-shop-frontend-2.png)](./img/hipster-shop-frontend-2.png) | +| [![Screenshot of store homepage](./docs/img/hipster-shop-frontend-1.png)](./docs/img/hipster-shop-frontend-1.png) | [![Screenshot of checkout screen](./docs/img/hipster-shop-frontend-2.png)](./docs/img/hipster-shop-frontend-2.png) | ## Service Architecture @@ -19,7 +19,7 @@ Stackdriver, gRPC** and similar cloud-native technologies nowadays. languages that talk to each other over gRPC. [![Architecture of -microservices](./img/architecture-diagram.png)](./img/architecture-diagram.png) +microservices](./docs/img/architecture-diagram.png)](./docs/img/architecture-diagram.png) Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). diff --git a/img/architecture-diagram.png b/docs/img/architecture-diagram.png similarity index 100% rename from img/architecture-diagram.png rename to docs/img/architecture-diagram.png diff --git a/img/hipster-shop-frontend-1.png b/docs/img/hipster-shop-frontend-1.png similarity index 100% rename from img/hipster-shop-frontend-1.png rename to docs/img/hipster-shop-frontend-1.png diff --git a/img/hipster-shop-frontend-2.png b/docs/img/hipster-shop-frontend-2.png similarity index 100% rename from img/hipster-shop-frontend-2.png rename to docs/img/hipster-shop-frontend-2.png From 94ed247c0b28d07b7fcf905c7dbbb556a9a0bdac Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Thu, 3 Jan 2019 11:25:03 -0800 Subject: [PATCH 02/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9829ab3..2455a52 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ web-based e-commerce app called **“Hipster Shop”** where users can browse it add them to the cart, and purchase them. **Google uses this application to demonstrate Kubernetes, GKE, Istio, -Stackdriver, gRPC** and similar cloud-native technologies nowadays. +Stackdriver, gRPC, OpenCensus** and similar cloud-native technologies. ## Screenshots From d944092100696aa4a5974ef5c2e710547a824622 Mon Sep 17 00:00:00 2001 From: rghetia Date: Thu, 3 Jan 2019 12:56:06 -0800 Subject: [PATCH 03/23] Add Jaeger support for Adservice. (#111) This is the first service that exports to jaeger. Others to follow. Requires jaeger to be instantiated using - helm install --name jaeger stable/jaeger-operator - kubectl apply -f jaeger.yaml === jaeger.yaml Content === apiVersion: io.jaegertracing/v1alpha1 kind: Jaeger metadata: name: jaeger Above steps will be added to README in subsequent PR. --- kubernetes-manifests/adservice.yaml | 2 ++ src/adservice/build.gradle | 4 +-- .../src/main/java/hipstershop/AdService.java | 31 +++++++++++++------ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/kubernetes-manifests/adservice.yaml b/kubernetes-manifests/adservice.yaml index dc0879f..0329a72 100644 --- a/kubernetes-manifests/adservice.yaml +++ b/kubernetes-manifests/adservice.yaml @@ -31,6 +31,8 @@ spec: env: - name: PORT value: "9555" + - name: JAEGER_ENABLED + value: "false" resources: requests: cpu: 200m diff --git a/src/adservice/build.gradle b/src/adservice/build.gradle index 59b0313..7450826 100644 --- a/src/adservice/build.gradle +++ b/src/adservice/build.gradle @@ -28,7 +28,6 @@ version = "0.1.0-SNAPSHOT" // CURRENT_OPENCENSUS_VERSION def opencensusVersion = "0.17.0" // LATEST_OPENCENSUS_RELEASE_VERSION def grpcVersion = "1.15.0" // CURRENT_GRPC_VERSION def jacksonVersion = "2.9.6" -def prometheusVersion = "0.3.0" tasks.withType(JavaCompile) { sourceCompatibility = '1.8' @@ -45,7 +44,7 @@ dependencies { compile fileTree(dir: offlineCompile, include: '*.jar') } else { compile "com.google.api.grpc:proto-google-common-protos:1.11.0", - "io.opencensus:opencensus-exporter-stats-prometheus:${opencensusVersion}", + "io.opencensus:opencensus-exporter-trace-jaeger:${opencensusVersion}", "io.opencensus:opencensus-exporter-stats-stackdriver:${opencensusVersion}", "io.opencensus:opencensus-exporter-trace-stackdriver:${opencensusVersion}", "io.opencensus:opencensus-exporter-trace-logging:${opencensusVersion}", @@ -53,7 +52,6 @@ dependencies { "io.grpc:grpc-stub:${grpcVersion}", "io.grpc:grpc-netty:${grpcVersion}", "io.grpc:grpc-services:${grpcVersion}", - "io.prometheus:simpleclient_httpserver:${prometheusVersion}", "org.apache.logging.log4j:log4j-core:2.11.1" runtime "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}", diff --git a/src/adservice/src/main/java/hipstershop/AdService.java b/src/adservice/src/main/java/hipstershop/AdService.java index 1224efe..b5d21a6 100644 --- a/src/adservice/src/main/java/hipstershop/AdService.java +++ b/src/adservice/src/main/java/hipstershop/AdService.java @@ -31,7 +31,7 @@ import io.grpc.services.*; import io.opencensus.common.Duration; import io.opencensus.common.Scope; import io.opencensus.contrib.grpc.metrics.RpcViews; -import io.opencensus.exporter.stats.prometheus.PrometheusStatsCollector; +import io.opencensus.exporter.trace.jaeger.JaegerTraceExporter; import io.opencensus.exporter.trace.logging.LoggingTraceExporter; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; @@ -46,7 +46,6 @@ import io.opencensus.trace.samplers.Samplers; import java.io.IOException; import java.util.Collection; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; @@ -201,11 +200,6 @@ public class AdService { public static void initStackdriver() { logger.info("Initialize StackDriver"); - // Registers all RPC views. - RpcViews.registerAllViews(); - - // Registers logging trace exporter. - LoggingTraceExporter.register(); long sleepTime = 10; /* seconds */ int maxAttempts = 5; boolean statsExporterRegistered = false; @@ -243,18 +237,35 @@ public class AdService { logger.info("StackDriver initialization complete."); } + static void initJaeger() { + boolean enabled = Boolean.parseBoolean(System.getenv("JAEGER_ENABLED")); + if (enabled) { + // Register Jaeger Tracing. + JaegerTraceExporter.createAndRegister("http://jaeger-collector:14268/api/traces", "adservice"); + logger.info("Jaeger initialization complete."); + } else { + logger.info("Jaeger initialization disabled."); + } + } + /** Main launches the server from the command line. */ public static void main(String[] args) throws IOException, InterruptedException { // Add final keyword to pass checkStyle. + // Registers all RPC views. + RpcViews.registerAllViews(); + + // Registers logging trace exporter. + LoggingTraceExporter.register(); + new Thread( new Runnable() { public void run(){ initStackdriver(); } }).start(); - // Register Prometheus exporters and export metrics to a Prometheus HTTPServer. - PrometheusStatsCollector.createAndRegister(); + // Register Jaeger + initJaeger(); // Start the RPC server. You shouldn't see any output from gRPC before this. logger.info("AdService starting."); @@ -262,4 +273,4 @@ public class AdService { service.start(); service.blockUntilShutdown(); } -} \ No newline at end of file +} From 27e1b03c26b0397cf451c7d4d9bce52dfe8b6fd2 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Tue, 8 Jan 2019 10:51:36 -0800 Subject: [PATCH 04/23] Add conference videos --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 2455a52..70b2046 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,14 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). curl -v "http://$INGRESS_HOST" +## Conferences featuring Hipster Shop + +- [Google Cloud Next'18 London – Keynote](https://youtu.be/nIq2pkNcfEI?t=3071) showing Stackdriver Incident Response Management +- Google Cloud Next'18 SF + - [Day 1 Keynote](https://youtu.be/vJ9OaAqfxo4?t=2416) showing GKE On-Prem + - [Day 3 – Keynote](https://youtu.be/JQPOPV_VH5w?t=815) showing Stackdriver APM (Tracing, Code Search, Profiler, Google Cloud Build) + - [Introduction to Service Management with Istio](https://www.youtube.com/watch?v=wCJrdKdD6UM&feature=youtu.be&t=586) + --- **Note to fellow Googlers:** Please fill out the form at From 85c7131d43b02ea7c0c777354ee1bd5c538de202 Mon Sep 17 00:00:00 2001 From: rghetia Date: Tue, 8 Jan 2019 11:08:20 -0800 Subject: [PATCH 05/23] Add jaeger support for frontend. (#113) --- kubernetes-manifests/frontend.yaml | 2 + src/frontend/Gopkg.lock | 111 +++++++++++++++++++---------- src/frontend/main.go | 36 +++++++++- 3 files changed, 111 insertions(+), 38 deletions(-) diff --git a/kubernetes-manifests/frontend.yaml b/kubernetes-manifests/frontend.yaml index e4b8e89..c83a1a7 100644 --- a/kubernetes-manifests/frontend.yaml +++ b/kubernetes-manifests/frontend.yaml @@ -58,6 +58,8 @@ spec: value: "checkoutservice:5050" - name: AD_SERVICE_ADDR value: "adservice:9555" + # - name: JAEGER_SERVICE_ADDR + # value: "jaeger-collector:14268" resources: requests: cpu: 100m diff --git a/src/frontend/Gopkg.lock b/src/frontend/Gopkg.lock index fa48498..53289b0 100644 --- a/src/frontend/Gopkg.lock +++ b/src/frontend/Gopkg.lock @@ -16,16 +16,33 @@ version = "v0.27.0" [[projects]] - digest = "1:9fe70def8f0ceb3d455a0acad9dadd6632287cdbf9c8c2ea50e8dabe2ade40c4" + digest = "1:4b96dcd8534bc6450a922bd16a76360ba3381f0d1daf40abbaec91c053fbfeb5" name = "contrib.go.opencensus.io/exporter/stackdriver" - packages = [ - ".", - "propagation", - ] + packages = ["."] pruneopts = "UT" revision = "37aa2801fbf0205003e15636096ebf0373510288" version = "v0.5.0" +[[projects]] + branch = "master" + digest = "1:1d8f3cb93c42c5652bb509fde29ecdd1feede9334e355e8bbdc0f9f95b40c254" + name = "git.apache.org/thrift.git" + packages = ["lib/go/thrift"] + pruneopts = "UT" + revision = "6503043bc42ab96da14c25f3aee2bb4add719774" + source = "github.com/apache/thrift" + +[[projects]] + branch = "master" + digest = "1:6cbe7676244a1429f4c22601f799d377a70449469ef636f91d992d719b559ff3" + name = "github.com/GoogleCloudPlatform/microservices-demo" + packages = [ + "src/frontend/genproto", + "src/frontend/money", + ] + pruneopts = "UT" + revision = "d944092100696aa4a5974ef5c2e710547a824622" + [[projects]] digest = "1:72856926f8208767b837bf51e3373f49139f61889b67dc7fd3c2a0fd711e3f7a" name = "github.com/golang/protobuf" @@ -50,23 +67,26 @@ name = "github.com/google/pprof" packages = ["profile"] pruneopts = "UT" - revision = "84b7d314e22c8d12334e52726f68517973b6027b" + revision = "3ea8567a2e5752420fc544d2e2ad249721768934" [[projects]] - digest = "1:3a26588bc48b96825977c1b3df964f8fd842cd6860cc26370588d3563433cf11" + digest = "1:236d7e1bdb50d8f68559af37dbcf9d142d56b431c9b2176d41e2a009b664cda8" name = "github.com/google/uuid" packages = ["."] pruneopts = "UT" - revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494" - version = "v1.0.0" + revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8" + version = "v1.1.0" [[projects]] - digest = "1:e145e9710a10bc114a6d3e2738aadf8de146adaa031854ffdf7bbfe15da85e63" + digest = "1:cd9864c6366515827a759931746738ede6079faa08df9c584596370d6add135c" name = "github.com/googleapis/gax-go" - packages = ["."] + packages = [ + ".", + "v2", + ] pruneopts = "UT" - revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f" - version = "v2.0.0" + revision = "c8a15bac9b9fe955bd9f900272f9a306465d28cf" + version = "v2.0.3" [[projects]] digest = "1:c79fb010be38a59d657c48c6ba1d003a8aa651fa56b579d959d74573b7dff8e1" @@ -84,6 +104,14 @@ revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" version = "v1.6.2" +[[projects]] + digest = "1:0a69a1c0db3591fcefb47f115b224592c8dfa4368b7ba9fae509d5e16cdc95c8" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "UT" + revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242" + version = "v1.0.1" + [[projects]] digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" name = "github.com/pkg/errors" @@ -93,18 +121,20 @@ version = "v0.8.0" [[projects]] - digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" + digest = "1:69b1cc331fca23d702bd72f860c6a647afd0aa9fcbc1d0659b1365e26546dd70" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" - revision = "3e01752db0189b9157070a0e1668a620f9a85da2" - version = "v1.0.6" + revision = "bcd833dfe83d3cebad139e4a29ed79cb2318bf95" + version = "v1.2.0" [[projects]] - digest = "1:1bb914cfb78f68f488a91cd7872d3d06a5f83c5bbacf0296dbef44e120b00a91" + digest = "1:a5154dfd6b37bef5a3eab759e13296348e639dc8c7604f538368167782b08ccd" name = "go.opencensus.io" packages = [ ".", + "exporter/jaeger", + "exporter/jaeger/internal/gen-go/jaeger", "internal", "internal/tagencoding", "plugin/ocgrpc", @@ -125,15 +155,15 @@ [[projects]] branch = "master" - digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" + digest = "1:38f553aff0273ad6f367cb0a0f8b6eecbaef8dc6cb8b50e57b6a81c1d5b1e332" name = "golang.org/x/crypto" packages = ["ssh/terminal"] pruneopts = "UT" - revision = "0e37d006457bf46f9e6692014ba72ef82c33022c" + revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" [[projects]] branch = "master" - digest = "1:1c14517b2f106c61d75006199b46a46576058661d469658cb0f90739919641d2" + digest = "1:8ecb828bb550a8c6b7d75b8261a42c369461311616ebe5451966d067f5f993bf" name = "golang.org/x/net" packages = [ "context", @@ -146,11 +176,11 @@ "trace", ] pruneopts = "UT" - revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2" + revision = "927f97764cc334a6575f4b7a1584a147864d5723" [[projects]] branch = "master" - digest = "1:f645667d687fc8bf228865a2c5455824ef05bad08841e673673ef2bb89ac5b90" + digest = "1:23443edff0740e348959763085df98400dcfd85528d7771bb0ce9f5f2754ff4a" name = "golang.org/x/oauth2" packages = [ ".", @@ -160,26 +190,26 @@ "jwt", ] pruneopts = "UT" - revision = "d2e6202438beef2727060aa7cabdd924d92ebfd9" + revision = "d668ce993890a79bda886613ee587a69dd5da7a6" [[projects]] branch = "master" - digest = "1:e0140c0c868c6e0f01c0380865194592c011fe521d6e12d78bfd33e756fe018a" + digest = "1:75515eedc0dc2cb0b40372008b616fa2841d831c63eedd403285ff286c593295" name = "golang.org/x/sync" packages = ["semaphore"] pruneopts = "UT" - revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + revision = "37e7f081c4d4c64e13b10787722085407fe5d15f" [[projects]] branch = "master" - digest = "1:374fc90fcb026e9a367e3fad29e988e5dd944b68ca3f24a184d77abc5307dda4" + digest = "1:191cccd950a4aeadb60306062f2bdc2f924d750d0156ec6c691b17211bfd7349" name = "golang.org/x/sys" packages = [ "unix", "windows", ] pruneopts = "UT" - revision = "d0be0721c37eeb5299f245a996a483160fc36940" + revision = "82a175fd1598e8a172e58ebdf5ed262bb29129e5" [[projects]] digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" @@ -206,7 +236,7 @@ [[projects]] branch = "master" - digest = "1:e9e388241f9f0f02000dddaeeb91153d53f0cd09dcec33879cc7e043a2e65d75" + digest = "1:2e81813e8e072aa700e101369890e55539729d817d32dbc3fab228d6b40c4d83" name = "google.golang.org/api" packages = [ "googleapi/transport", @@ -217,12 +247,13 @@ "transport", "transport/grpc", "transport/http", + "transport/http/internal/propagation", ] pruneopts = "UT" - revision = "7ca32eb868bf53ea2fc406698eb98583a8073d19" + revision = "19e022d8cf43ce81f046bae8cc18c5397cc7732f" [[projects]] - digest = "1:26619fcd2452b4044174d26acd8b09c09dffee9a1c3a22d2383b873aa9a0131f" + digest = "1:c4eaa5f79d36f76ef4bd0c4f96e36bc1b7b5a359528d1267f0cb7a5d58b7b5bb" name = "google.golang.org/appengine" packages = [ ".", @@ -239,12 +270,12 @@ "urlfetch", ] pruneopts = "UT" - revision = "b1f26356af11148e710935ed1ac8a7f5702c7612" - version = "v1.1.0" + revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" + version = "v1.4.0" [[projects]] branch = "master" - digest = "1:7a3da01a8f840fbbef1f027dc091ae52a29c6ab9374e126b6bdc5bf3b0ff2687" + digest = "1:3552e7267a98c605e6cbfe6b03c34589265ab72e6078b95fb2e10e0cb44f5cc8" name = "google.golang.org/genproto" packages = [ "googleapis/api/annotations", @@ -260,28 +291,33 @@ "protobuf/field_mask", ] pruneopts = "UT" - revision = "36d5787dc5356b711fe8f3040271aaf51c35955b" + revision = "bd9b4fb69e2ffd37621a6caa54dcbead29b546f2" [[projects]] - digest = "1:4ad047d772a7d4841687df9534a767a9e243885ed6d7d2ced6af5995a5dc44b3" + digest = "1:3fc54ad826c0183f803bb97e0927dfc05fcb7b7a6ddabed646ee8184d861fa9b" name = "google.golang.org/grpc" packages = [ ".", "balancer", "balancer/base", "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", "codes", "connectivity", "credentials", + "credentials/internal", "credentials/oauth", "encoding", "encoding/proto", "grpclog", "internal", "internal/backoff", + "internal/binarylog", "internal/channelz", "internal/envconfig", "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", "internal/transport", "keepalive", "metadata", @@ -295,8 +331,8 @@ "tap", ] pruneopts = "UT" - revision = "8dea3dc473e90c8179e519d91302d0597c0ca1d1" - version = "v1.15.0" + revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8" + version = "v1.17.0" [solve-meta] analyzer-name = "dep" @@ -311,6 +347,7 @@ "github.com/gorilla/mux", "github.com/pkg/errors", "github.com/sirupsen/logrus", + "go.opencensus.io/exporter/jaeger", "go.opencensus.io/plugin/ocgrpc", "go.opencensus.io/plugin/ochttp", "go.opencensus.io/plugin/ochttp/propagation/b3", diff --git a/src/frontend/main.go b/src/frontend/main.go index 5046432..61ac8ce 100644 --- a/src/frontend/main.go +++ b/src/frontend/main.go @@ -26,6 +26,7 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "go.opencensus.io/exporter/jaeger" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp/propagation/b3" @@ -142,6 +143,29 @@ func main() { log.Fatal(http.ListenAndServe(addr+":"+srvPort, handler)) } +func initJaegerTracing(log logrus.FieldLogger) { + + svcAddr := os.Getenv("JAEGER_SERVICE_ADDR") + if svcAddr == "" { + log.Info("jaeger initialization disabled.") + return + } + + // Register the Jaeger exporter to be able to retrieve + // the collected spans. + exporter, err := jaeger.NewExporter(jaeger.Options{ + Endpoint: fmt.Sprintf("http://%s", svcAddr), + Process: jaeger.Process{ + ServiceName: "frontend", + }, + }) + if err != nil { + log.Fatal(err) + } + trace.RegisterExporter(exporter) + log.Info("jaeger initialization completed.") +} + func initStats(log logrus.FieldLogger, exporter *stackdriver.Exporter) { view.SetReportingPeriod(60 * time.Second) view.RegisterExporter(exporter) @@ -158,16 +182,26 @@ func initStats(log logrus.FieldLogger, exporter *stackdriver.Exporter) { } func initTracing(log logrus.FieldLogger) { + // This is a demo app with low QPS. trace.AlwaysSample() is used here + // to make sure traces are available for observation and analysis. + // In a production environment or high QPS setup please use + // trace.ProbabilitySampler set at the desired probability. + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + + initJaegerTracing(log) + // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. for i := 1; i <= 3; i++ { log = log.WithField("retry", i) exporter, err := stackdriver.NewExporter(stackdriver.Options{}) if err != nil { + // log.Warnf is used since there are multiple backends (stackdriver & jaeger) + // to store the traces. In production setup most likely you would use only one backend. + // In that case you should use log.Fatalf. log.Warnf("failed to initialize stackdriver exporter: %+v", err) } else { trace.RegisterExporter(exporter) - trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) log.Info("registered stackdriver tracing") // Register the views to collect server stats. From 33ca3b63d82698035ffc1230dcb650885a005197 Mon Sep 17 00:00:00 2001 From: rghetia Date: Wed, 9 Jan 2019 05:42:54 -0800 Subject: [PATCH 06/23] Unify jaeger enabling method for adservice. (#115) --- kubernetes-manifests/adservice.yaml | 8 ++++---- src/adservice/src/main/java/hipstershop/AdService.java | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/kubernetes-manifests/adservice.yaml b/kubernetes-manifests/adservice.yaml index 0329a72..76f35ae 100644 --- a/kubernetes-manifests/adservice.yaml +++ b/kubernetes-manifests/adservice.yaml @@ -31,8 +31,8 @@ spec: env: - name: PORT value: "9555" - - name: JAEGER_ENABLED - value: "false" + #- name: JAEGER_SERVICE_ADDR + # value: "jaeger-collector:14268" resources: requests: cpu: 200m @@ -42,12 +42,12 @@ spec: memory: 300Mi readinessProbe: initialDelaySeconds: 20 - periodSeconds: 5 + periodSeconds: 15 exec: command: ["/bin/grpc_health_probe", "-addr=:9555"] livenessProbe: initialDelaySeconds: 20 - periodSeconds: 5 + periodSeconds: 15 exec: command: ["/bin/grpc_health_probe", "-addr=:9555"] --- diff --git a/src/adservice/src/main/java/hipstershop/AdService.java b/src/adservice/src/main/java/hipstershop/AdService.java index b5d21a6..7320764 100644 --- a/src/adservice/src/main/java/hipstershop/AdService.java +++ b/src/adservice/src/main/java/hipstershop/AdService.java @@ -238,10 +238,11 @@ public class AdService { } static void initJaeger() { - boolean enabled = Boolean.parseBoolean(System.getenv("JAEGER_ENABLED")); - if (enabled) { + String jaegerAddr = System.getenv("JAEGER_SERVICE_ADDR"); + if (jaegerAddr != null && !jaegerAddr.isEmpty()) { + String jaegerUrl = String.format("http://%s/api/traces", jaegerAddr); // Register Jaeger Tracing. - JaegerTraceExporter.createAndRegister("http://jaeger-collector:14268/api/traces", "adservice"); + JaegerTraceExporter.createAndRegister(jaegerUrl, "adservice"); logger.info("Jaeger initialization complete."); } else { logger.info("Jaeger initialization disabled."); From 2b074f7ff3d6dfbd4220dc29c1abd8ea052012df Mon Sep 17 00:00:00 2001 From: rghetia Date: Thu, 10 Jan 2019 10:25:07 -0800 Subject: [PATCH 07/23] Add jaeger export to checkoutservice. (#116) * Add jaeger export to checkoutservice. * fix review comments. --- kubernetes-manifests/checkoutservice.yaml | 2 + src/checkoutservice/Gopkg.lock | 153 +++++++++++++++++----- src/checkoutservice/main.go | 31 ++++- 3 files changed, 149 insertions(+), 37 deletions(-) diff --git a/kubernetes-manifests/checkoutservice.yaml b/kubernetes-manifests/checkoutservice.yaml index aec9591..1b537df 100644 --- a/kubernetes-manifests/checkoutservice.yaml +++ b/kubernetes-manifests/checkoutservice.yaml @@ -46,6 +46,8 @@ spec: value: "currencyservice:7000" - name: CART_SERVICE_ADDR value: "cartservice:7070" + # - name: JAEGER_SERVICE_ADDR + # value: "jaeger-collector:14268" resources: requests: cpu: 100m diff --git a/src/checkoutservice/Gopkg.lock b/src/checkoutservice/Gopkg.lock index 364e527..273cb71 100644 --- a/src/checkoutservice/Gopkg.lock +++ b/src/checkoutservice/Gopkg.lock @@ -2,27 +2,49 @@ [[projects]] + digest = "1:467af0aad47996b25b838d6f14c8371123a8a76ec239020a6c5894e1f8f60272" name = "cloud.google.com/go" packages = [ "compute/metadata", "internal/version", "monitoring/apiv3", "profiler", - "trace/apiv2" + "trace/apiv2", ] + pruneopts = "UT" revision = "c728a003b238b26cef9ab6753a5dc424b331c3ad" version = "v0.27.0" [[projects]] + digest = "1:4b96dcd8534bc6450a922bd16a76360ba3381f0d1daf40abbaec91c053fbfeb5" name = "contrib.go.opencensus.io/exporter/stackdriver" - packages = [ - ".", - "propagation" - ] + packages = ["."] + pruneopts = "UT" revision = "37aa2801fbf0205003e15636096ebf0373510288" version = "v0.5.0" [[projects]] + branch = "master" + digest = "1:1d8f3cb93c42c5652bb509fde29ecdd1feede9334e355e8bbdc0f9f95b40c254" + name = "git.apache.org/thrift.git" + packages = ["lib/go/thrift"] + pruneopts = "UT" + revision = "a5df39032ca206e2e6a9ec975147e81746d9a255" + source = "github.com/apache/thrift" + +[[projects]] + branch = "master" + digest = "1:f6bdf3d8d3cbb2f98d3ebaa66b3ac25166a06830027ece7d592d9ea09dedf504" + name = "github.com/GoogleCloudPlatform/microservices-demo" + packages = [ + "src/checkoutservice/genproto", + "src/checkoutservice/money", + ] + pruneopts = "UT" + revision = "33ca3b63d82698035ffc1230dcb650885a005197" + +[[projects]] + digest = "1:72856926f8208767b837bf51e3373f49139f61889b67dc7fd3c2a0fd711e3f7a" name = "github.com/golang/protobuf" packages = [ "proto", @@ -33,41 +55,62 @@ "ptypes/empty", "ptypes/struct", "ptypes/timestamp", - "ptypes/wrappers" + "ptypes/wrappers", ] + pruneopts = "UT" revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" [[projects]] branch = "master" + digest = "1:089d56c0adb79140365b5c86815ce97233986da6f3a525c6b706773e4b83876f" name = "github.com/google/pprof" packages = ["profile"] - revision = "e027b505a088ac3c68c339a1d7ce7724bf34538b" + pruneopts = "UT" + revision = "985cf9b4fdd2b479e4563ec57352005f69d5f9f6" [[projects]] + digest = "1:236d7e1bdb50d8f68559af37dbcf9d142d56b431c9b2176d41e2a009b664cda8" name = "github.com/google/uuid" packages = ["."] - revision = "d460ce9f8df2e77fb1ba55ca87fafed96c607494" - version = "v1.0.0" + pruneopts = "UT" + revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8" + version = "v1.1.0" [[projects]] + digest = "1:cd9864c6366515827a759931746738ede6079faa08df9c584596370d6add135c" name = "github.com/googleapis/gax-go" - packages = ["."] - revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f" - version = "v2.0.0" + packages = [ + ".", + "v2", + ] + pruneopts = "UT" + revision = "c8a15bac9b9fe955bd9f900272f9a306465d28cf" + version = "v2.0.3" [[projects]] - digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" + digest = "1:0a69a1c0db3591fcefb47f115b224592c8dfa4368b7ba9fae509d5e16cdc95c8" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "UT" + revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242" + version = "v1.0.1" + +[[projects]] + digest = "1:87c2e02fb01c27060ccc5ba7c5a407cc91147726f8f40b70cceeedbc52b1f3a8" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" - revision = "3e01752db0189b9157070a0e1668a620f9a85da2" - version = "v1.0.6" + revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5" + version = "v1.3.0" [[projects]] + digest = "1:a5154dfd6b37bef5a3eab759e13296348e639dc8c7604f538368167782b08ccd" name = "go.opencensus.io" packages = [ ".", + "exporter/jaeger", + "exporter/jaeger/internal/gen-go/jaeger", "internal", "internal/tagencoding", "plugin/ocgrpc", @@ -80,21 +123,23 @@ "trace", "trace/internal", "trace/propagation", - "trace/tracestate" + "trace/tracestate", ] + pruneopts = "UT" revision = "b11f239c032624b045c4c2bfd3d1287b4012ce89" version = "v0.16.0" [[projects]] branch = "master" - digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" + digest = "1:38f553aff0273ad6f367cb0a0f8b6eecbaef8dc6cb8b50e57b6a81c1d5b1e332" name = "golang.org/x/crypto" packages = ["ssh/terminal"] pruneopts = "UT" - revision = "0e37d006457bf46f9e6692014ba72ef82c33022c" + revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" [[projects]] branch = "master" + digest = "1:8ecb828bb550a8c6b7d75b8261a42c369461311616ebe5451966d067f5f993bf" name = "golang.org/x/net" packages = [ "context", @@ -104,35 +149,46 @@ "http2/hpack", "idna", "internal/timeseries", - "trace" + "trace", ] - revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2" + pruneopts = "UT" + revision = "1e06a53dbb7e2ed46e91183f219db23c6943c532" [[projects]] branch = "master" + digest = "1:23443edff0740e348959763085df98400dcfd85528d7771bb0ce9f5f2754ff4a" name = "golang.org/x/oauth2" packages = [ ".", "google", "internal", "jws", - "jwt" + "jwt", ] - revision = "d2e6202438beef2727060aa7cabdd924d92ebfd9" + pruneopts = "UT" + revision = "d668ce993890a79bda886613ee587a69dd5da7a6" [[projects]] branch = "master" + digest = "1:75515eedc0dc2cb0b40372008b616fa2841d831c63eedd403285ff286c593295" name = "golang.org/x/sync" packages = ["semaphore"] - revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + pruneopts = "UT" + revision = "37e7f081c4d4c64e13b10787722085407fe5d15f" [[projects]] branch = "master" + digest = "1:12ef3ef293a3a3930a82e5f38a3c45a1b03a9ed72dedc192d90e89d59b1f13a5" name = "golang.org/x/sys" - packages = ["unix"] - revision = "1561086e645b2809fb9f8a1e2a38160bf8d53bf4" + packages = [ + "unix", + "windows", + ] + pruneopts = "UT" + revision = "7fbe1cd0fcc20051e1fcb87fbabec4a1bacaaeba" [[projects]] + digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" name = "golang.org/x/text" packages = [ "collate", @@ -148,13 +204,15 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:b74a0ae8b7755bf9cdadead4dea674d76517cd2fea7bd482a6a46bb6b3ce085b" name = "google.golang.org/api" packages = [ "googleapi/transport", @@ -164,11 +222,14 @@ "support/bundler", "transport", "transport/grpc", - "transport/http" + "transport/http", + "transport/http/internal/propagation", ] - revision = "19ff8768a5c0b8e46ea281065664787eefc24121" + pruneopts = "UT" + revision = "032faecc3e7e2c445ec37a1b1ec4e5a3016d96f2" [[projects]] + digest = "1:c4eaa5f79d36f76ef4bd0c4f96e36bc1b7b5a359528d1267f0cb7a5d58b7b5bb" name = "google.golang.org/appengine" packages = [ ".", @@ -182,13 +243,15 @@ "internal/socket", "internal/urlfetch", "socket", - "urlfetch" + "urlfetch", ] - revision = "ae0ab99deb4dc413a2b4bd6c8bdd0eb67f1e4d06" - version = "v1.2.0" + pruneopts = "UT" + revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" + version = "v1.4.0" [[projects]] branch = "master" + digest = "1:3552e7267a98c605e6cbfe6b03c34589265ab72e6078b95fb2e10e0cb44f5cc8" name = "google.golang.org/genproto" packages = [ "googleapis/api/annotations", @@ -201,11 +264,13 @@ "googleapis/monitoring/v3", "googleapis/rpc/errdetails", "googleapis/rpc/status", - "protobuf/field_mask" + "protobuf/field_mask", ] - revision = "c3f76f3b92d1ffa4c58a9ff842a58b8877655e0f" + pruneopts = "UT" + revision = "ae2f86662275e140f395167f1dab7081a5bd5fa8" [[projects]] + digest = "1:6497ab07ec89179db8d5a563d33635be04ceffaa29007a3ae74b9f15f4d3068e" name = "google.golang.org/grpc" packages = [ ".", @@ -235,14 +300,32 @@ "resolver/passthrough", "stats", "status", - "tap" + "tap", ] + pruneopts = "UT" revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455" version = "v1.14.0" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "0c103e5e32d8ecdd70048b6d91e02b910e30bad54f06b230a9664f585433a669" + input-imports = [ + "cloud.google.com/go/profiler", + "contrib.go.opencensus.io/exporter/stackdriver", + "github.com/GoogleCloudPlatform/microservices-demo/src/checkoutservice/genproto", + "github.com/GoogleCloudPlatform/microservices-demo/src/checkoutservice/money", + "github.com/golang/protobuf/proto", + "github.com/google/uuid", + "github.com/sirupsen/logrus", + "go.opencensus.io/exporter/jaeger", + "go.opencensus.io/plugin/ocgrpc", + "go.opencensus.io/stats/view", + "go.opencensus.io/trace", + "golang.org/x/net/context", + "google.golang.org/grpc", + "google.golang.org/grpc/codes", + "google.golang.org/grpc/health/grpc_health_v1", + "google.golang.org/grpc/status", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/src/checkoutservice/main.go b/src/checkoutservice/main.go index 990b004..fdf1c83 100644 --- a/src/checkoutservice/main.go +++ b/src/checkoutservice/main.go @@ -25,6 +25,7 @@ import ( "contrib.go.opencensus.io/exporter/stackdriver" "github.com/google/uuid" "github.com/sirupsen/logrus" + "go.opencensus.io/exporter/jaeger" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/stats/view" "go.opencensus.io/trace" @@ -98,6 +99,28 @@ func main() { log.Fatal(err) } +func initJaegerTracing() { + svcAddr := os.Getenv("JAEGER_SERVICE_ADDR") + if svcAddr == "" { + log.Info("jaeger initialization disabled.") + return + } + + // Register the Jaeger exporter to be able to retrieve + // the collected spans. + exporter, err := jaeger.NewExporter(jaeger.Options{ + Endpoint: fmt.Sprintf("http://%s", svcAddr), + Process: jaeger.Process{ + ServiceName: "checkoutservice", + }, + }) + if err != nil { + log.Fatal(err) + } + trace.RegisterExporter(exporter) + log.Info("jaeger initialization completed.") +} + func initStats(exporter *stackdriver.Exporter) { view.SetReportingPeriod(60 * time.Second) view.RegisterExporter(exporter) @@ -108,7 +131,7 @@ func initStats(exporter *stackdriver.Exporter) { } } -func initTracing() { +func initStackDriverTracing() { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. for i := 1; i <= 3; i++ { @@ -117,7 +140,6 @@ func initTracing() { log.Infof("failed to initialize stackdriver exporter: %+v", err) } else { trace.RegisterExporter(exporter) - trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) log.Info("registered stackdriver tracing") // Register the views to collect server stats. @@ -131,6 +153,11 @@ func initTracing() { log.Warn("could not initialize stackdriver exporter after retrying, giving up") } +func initTracing() { + initJaegerTracing() + initStackDriverTracing() +} + func initProfiling(service, version string) { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. From 823e993104bb78d584153d4087e29159bdadb5aa Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Thu, 10 Jan 2019 10:35:17 -0800 Subject: [PATCH 08/23] update skaffold manifest to clear warnings (#117) Signed-off-by: Ahmet Alp Balkan --- .travis.yml | 2 +- skaffold.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50a839e..124f5dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ services: - docker install: -- curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v0.18.0/skaffold-linux-amd64 +- curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v0.20.0/skaffold-linux-amd64 - chmod +x skaffold - sudo mv skaffold /usr/local/bin diff --git a/skaffold.yaml b/skaffold.yaml index eb2f9c4..5c880dd 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: skaffold/v1alpha5 +apiVersion: skaffold/v1beta2 kind: Config build: artifacts: From aaf2f8717dc3c7ca17b42fea8706f27667c20244 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Fri, 11 Jan 2019 16:59:27 -0600 Subject: [PATCH 09/23] Simplify package management for python services (#120) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR does a few things: 1. **Removes unnecessary Python dependencies currently being installed for `emailservice`** There are quite a few packages being installed that aren't actual dependencies. 2. **Removes a number of related, also unnecessary system-level dependencies for `emailservice`** These were a result of the Python dependencies that are unnecessary. 3. **Pins all of the sub-dependencies for `loadgenerator`** This is good practice to ensure that things don't break one day in the future when a newer version of an unpinned sub-dependenency is released. 4. **Compile all Python dependencies from `requirements.in` files** This is mostly bookkeeping. It allows us to only specify the top-level dependencies we care about in the requirements.in files, which are then compiled to frozen dependencies in the requirements.txt files. This ensures that we only install the dependencies we need, and that we're not missing any unpinned sub-dependencies. It also makes it more clear where our sub-dependencies are coming from. 5. **Switch to -slim images from -alpine** Python's built distribution format (wheel) is incompatible with alpine-based images, causing dependencies like `grpcio` to be compiled from scratch, rather than from a pre-built wheel. This should improve or possibly fi​x #58, while keeping the image size roughly the same: ``` emailservice latest d1b818eabe05 6 seconds ago 286MB loadgenerator latest 4d9b5acbfbbb 6 seconds ago 125MB ``` --- src/emailservice/Dockerfile | 23 +++-------- src/emailservice/requirements.in | 6 +++ src/emailservice/requirements.txt | 64 ++++++++++++------------------ src/loadgenerator/Dockerfile | 12 ++---- src/loadgenerator/requirements.in | 1 + src/loadgenerator/requirements.txt | 23 ++++++++++- 6 files changed, 64 insertions(+), 65 deletions(-) create mode 100644 src/emailservice/requirements.in create mode 100644 src/loadgenerator/requirements.in diff --git a/src/emailservice/Dockerfile b/src/emailservice/Dockerfile index 0bb5695..b4b2a94 100644 --- a/src/emailservice/Dockerfile +++ b/src/emailservice/Dockerfile @@ -1,21 +1,7 @@ -FROM python:3-alpine as base +FROM python:3-slim as base FROM base as builder -# gRPC and app deps -RUN apk add --update --no-cache \ - gcc \ - linux-headers \ - make \ - musl-dev \ - python-dev \ - g++ \ - # App Deps - cairo-dev \ - cairo \ - openssl-dev \ - gobject-introspection-dev - # get packages COPY requirements.txt . RUN pip install -r requirements.txt @@ -25,6 +11,10 @@ FROM base as final # Enable unbuffered logging ENV PYTHONUNBUFFERED=1 +RUN apt-get -qq update \ + && apt-get install -y --no-install-recommends \ + wget + # Download the grpc health probe RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ @@ -35,9 +25,6 @@ WORKDIR /email_server # Grab packages from builder COPY --from=builder /usr/local/lib/python3.7/ /usr/local/lib/python3.7/ -# Need libstdc++ for grpc -RUN apk add --no-cache libstdc++ - # Add the application COPY . . diff --git a/src/emailservice/requirements.in b/src/emailservice/requirements.in new file mode 100644 index 0000000..fa3f39c --- /dev/null +++ b/src/emailservice/requirements.in @@ -0,0 +1,6 @@ +google-api-core==1.6.0 +grpcio-health-checking==1.12.1 +grpcio==1.16.1 +jinja2==2.10 +opencensus[stackdriver]==0.1.10 +python-json-logger==0.1.9 diff --git a/src/emailservice/requirements.txt b/src/emailservice/requirements.txt index 65e7717..6a763b2 100644 --- a/src/emailservice/requirements.txt +++ b/src/emailservice/requirements.txt @@ -1,41 +1,29 @@ -asn1crypto==0.24.0 -cachetools==2.1.0 -certifi==2018.8.24 -cffi==1.11.5 -chardet==3.0.4 -configparser==3.5.0 -cryptography==2.3.1 -entrypoints==0.2.3 -enum34==1.1.6 -futures==3.1.1 -google-api-core==1.6.0 -google-auth==1.6.1 -google-cloud-core==0.29.0 -googleapis-common-protos==1.5.3 -grpc-google-iam-v1==0.11.4 -grpcio==1.16.1 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements.txt requirements.in +# +cachetools==3.0.0 # via google-auth +certifi==2018.11.29 # via requests +chardet==3.0.4 # via requests +google-api-core[grpc]==1.6.0 +google-auth==1.6.2 # via google-api-core +google-cloud-core==0.29.1 # via google-cloud-trace +google-cloud-trace==0.20.2 # via opencensus +googleapis-common-protos==1.5.5 # via google-api-core grpcio-health-checking==1.12.1 -grpcio-tools==1.12.1 -idna==2.7 -ipaddress==1.0.22 -jeepney==0.4 -Jinja2==2.10 -keyring==15.1.0 -keyrings.alt==3.1 -MarkupSafe==1.0 +grpcio==1.16.1 +idna==2.8 # via requests +jinja2==2.10 +markupsafe==1.1.0 # via jinja2 opencensus[stackdriver]==0.1.10 -protobuf==3.6.1 -pyasn1==0.4.4 -pyasn1-modules==0.2.2 -pycairo==1.17.1 -pycparser==2.19 -pycrypto==2.6.1 -PyGObject==3.30.1 +protobuf==3.6.1 # via google-api-core, googleapis-common-protos, grpcio-health-checking +pyasn1-modules==0.2.3 # via google-auth +pyasn1==0.4.5 # via pyasn1-modules, rsa python-json-logger==0.1.9 -pytz==2018.5 -pyxdg==0.26 -requests==2.20.0 -rsa==4.0 -SecretStorage==3.1.0 -six==1.11.0 -urllib3==1.23 +pytz==2018.9 # via google-api-core +requests==2.21.0 # via google-api-core +rsa==4.0 # via google-auth +six==1.12.0 # via google-api-core, google-auth, grpcio, protobuf +urllib3==1.24.1 # via requests diff --git a/src/loadgenerator/Dockerfile b/src/loadgenerator/Dockerfile index b0004cf..cebb98c 100644 --- a/src/loadgenerator/Dockerfile +++ b/src/loadgenerator/Dockerfile @@ -1,14 +1,10 @@ -FROM python:3-alpine as base +FROM python:3-slim as base FROM base as builder -RUN apk add --update --no-cache \ - gcc \ - linux-headers \ - make \ - musl-dev \ - python-dev \ - g++ +RUN apt-get -qq update \ + && apt-get install -y --no-install-recommends \ + g++ COPY requirements.txt . diff --git a/src/loadgenerator/requirements.in b/src/loadgenerator/requirements.in new file mode 100644 index 0000000..0b66c0a --- /dev/null +++ b/src/loadgenerator/requirements.in @@ -0,0 +1 @@ +locustio==0.8.1 diff --git a/src/loadgenerator/requirements.txt b/src/loadgenerator/requirements.txt index fa77c1c..ac69aca 100644 --- a/src/loadgenerator/requirements.txt +++ b/src/loadgenerator/requirements.txt @@ -1,2 +1,23 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements.txt requirements.in +# +certifi==2018.11.29 # via requests +chardet==3.0.4 # via requests +click==7.0 # via flask +flask==1.0.2 # via locustio +gevent==1.4.0 # via locustio +greenlet==0.4.15 # via gevent +idna==2.8 # via requests +itsdangerous==1.1.0 # via flask +jinja2==2.10 # via flask locustio==0.8.1 -pyzmq==17.0.0 +markupsafe==1.1.0 # via jinja2 +msgpack-python==0.5.6 # via locustio +pyzmq==17.0.0 # via locustio +requests==2.21.0 # via locustio +six==1.12.0 # via locustio +urllib3==1.24.1 # via requests +werkzeug==0.14.1 # via flask From 791a2822ad78c69cf5463ca700df7d346d7557c4 Mon Sep 17 00:00:00 2001 From: Dave Stanke Date: Mon, 14 Jan 2019 11:11:18 -0500 Subject: [PATCH 10/23] Update skaffold to v0.20.0 (to support newer skaffold config) (#123) --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index bf8aea7..2d031ce 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -11,7 +11,7 @@ steps: - id: 'Deploy application to cluster' - name: 'gcr.io/k8s-skaffold/skaffold:v0.18.0' + name: 'gcr.io/k8s-skaffold/skaffold:v0.20.0' entrypoint: 'bash' args: - '-c' From dcdd657d82440c2ca285448908ae89ad1191ad3e Mon Sep 17 00:00:00 2001 From: Oussema CHERNI Date: Mon, 14 Jan 2019 17:12:52 +0100 Subject: [PATCH 11/23] Fix broken doc link (#122) --- skaffold.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skaffold.yaml b/skaffold.yaml index 5c880dd..d39d561 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -19,7 +19,7 @@ build: # image tags are relative; to specify an image repo (e.g. GCR), you # must provide a "default repo" using one of the methods described # here: - # https://github.com/GoogleContainerTools/skaffold/blob/master/docs/concepts.adoc#2-push + # https://skaffold.dev/docs/concepts/#image-repository-handling - image: emailservice context: src/emailservice - image: productcatalogservice From 8deb49e5ecb2a8f16e02b507a175480a8d788163 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Mon, 14 Jan 2019 14:42:13 -0800 Subject: [PATCH 12/23] Update README.md --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 70b2046..2d285a3 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,11 @@ add them to the cart, and purchase them. **Google uses this application to demonstrate Kubernetes, GKE, Istio, Stackdriver, gRPC, OpenCensus** and similar cloud-native technologies. +> **Note to Googlers:** Please fill out the form at +[go/microservices-demo](http://go/microservices-demo) if you are using this +application. + + ## Screenshots | Home Page | Checkout Screen | @@ -184,8 +189,4 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). --- -**Note to fellow Googlers:** Please fill out the form at -[go/microservices-demo](http://go/microservices-demo) if you are using this -application. - -This is not an official Google project. +This is not an official Google project. From 10dfd04ab174cc680ed6ffef26cc4fb09ec40404 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Tue, 15 Jan 2019 10:14:33 -0800 Subject: [PATCH 13/23] README: use GKE add-on for Istio instructions (#121) I also edited some irrelevant parts of the file, don't mind those. Signed-off-by: Ahmet Alp Balkan --- README.md | 57 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2d285a3..e6d0828 100644 --- a/README.md +++ b/README.md @@ -80,15 +80,17 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). here](https://docs.docker.com/docker-for-mac/kubernetes/). - [skaffold](https://github.com/GoogleContainerTools/skaffold/#installation) -1. Launch “Docker for Desktop”. Go to Preferences and choose “Enable Kubernetes”. +1. Launch “Docker for Desktop”. Go to Preferences: + - choose “Enable Kubernetes”, + - set CPUs to at least 3, and Memory to at least 6.0 GiB -1. Run `kubectl get nodes` to verify you're connected to “Kubernetes on Docker”. +3. Run `kubectl get nodes` to verify you're connected to “Kubernetes on Docker”. -1. Run `skaffold run` (first time will be slow, it can take ~20-30 minutes). +4. Run `skaffold run` (first time will be slow, it can take ~20-30 minutes). This will build and deploy the application. If you need to rebuild the images automatically as you refactor he code, run `skaffold dev` command. -1. Run `kubectl get pods` to verify the Pods are ready and running. The +5. Run `kubectl get pods` to verify the Pods are ready and running. The application frontend should be available at http://localhost:80 on your machine. @@ -117,16 +119,18 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). 1. In the root of this repository, run `skaffold run --default-repo=gcr.io/[PROJECT_ID]`, where [PROJECT_ID] is your GCP project ID. - + This command: - builds the container images - pushes them to GCR - applies the `./kubernetes-manifests` deploying the application to Kubernetes. - **Troubleshooting:** If you get "No space left on device" error on Google Cloud Shell, - you can build the images on Google Cloud Build: - [Enable the Cloud Build API](https://console.cloud.google.com/flows/enableapi?apiid=cloudbuild.googleapis.com), then run `skaffold run -p gcb --default-repo=gcr.io/[PROJECT_ID]` instead. + **Troubleshooting:** If you get "No space left on device" error on Google + Cloud Shell, you can build the images on Google Cloud Build: [Enable the + Cloud Build + API](https://console.cloud.google.com/flows/enableapi?apiid=cloudbuild.googleapis.com), + then run `skaffold run -p gcb --default-repo=gcr.io/[PROJECT_ID]` instead. 1. Find the IP address of your application, then visit the application on your browser to confirm installation. @@ -139,38 +143,47 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). are seeing this, run `kubectl get service frontend-external -o=yaml | kubectl apply -f-` to trigger load balancer reconfiguration. -### (Optional) Deploying on a Istio-installed cluster +### (Optional) Deploying on a Istio-installed GKE cluster > **Note:** you followed GKE deployment steps above, run `skaffold delete` first > to delete what's deployed. -1. Create a GKE cluster. +1. Create a GKE cluster (described above). -2. Install Istio **without mutual TLS** authentication option. +2. Use [Istio on GKE add-on](https://cloud.google.com/istio/docs/istio-on-gke/installing) + to install Istio to your existing GKE cluster. - > (Optional) If you'd like to enable mTLS in the demo app, you need to - > make a few changes to the deployment manifests: + gcloud beta container clusters update demo \ + --zone=us-central1-a \ + --update-addons=Istio=ENABLED \ + --istio-config=auth=MTLS_PERMISSIVE + + > NOTE: If you need to enable `MTLS_STRICT` mode, you will need to update + > several manifest files: > > - `kubernetes-manifests/frontend.yaml`: delete "livenessProbe" and > "readinessProbe" fields. > - `kubernetes-manifests/loadgenerator.yaml`: delete "initContainers" field. -3. Install the automatic sidecar injection (annotate the `default` namespace +3. (Optional) Enable Stackdriver Tracing/Logging with Istio Stackdriver Adapter + by [following this guide](https://cloud.google.com/istio/docs/istio-on-gke/installing#enabling_tracing_and_logging). + +4. Install the automatic sidecar injection (annotate the `default` namespace with the label): kubectl label namespace default istio-injection=enabled -4. Apply the manifests in [`./istio-manifests`](./istio-manifests) directory. +5. Apply the manifests in [`./istio-manifests`](./istio-manifests) directory. kubectl apply -f ./istio-manifests This is required only once. -5. Deploy the application with `skaffold run --default-repo=gcr.io/[PROJECT_ID]`. +6. Deploy the application with `skaffold run --default-repo=gcr.io/[PROJECT_ID]`. -6. Run `kubectl get pods` to see pods are in a healthy and ready state. +7. Run `kubectl get pods` to see pods are in a healthy and ready state. -7. Find the IP address of your istio gateway Ingress or Service, and visit the +8. Find the IP address of your istio gateway Ingress or Service, and visit the application. INGRESS_HOST="$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" @@ -179,12 +192,14 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). curl -v "http://$INGRESS_HOST" -## Conferences featuring Hipster Shop +## Conferences featuring Hipster Shop -- [Google Cloud Next'18 London – Keynote](https://youtu.be/nIq2pkNcfEI?t=3071) showing Stackdriver Incident Response Management +- [Google Cloud Next'18 London – Keynote](https://youtu.be/nIq2pkNcfEI?t=3071) + showing Stackdriver Incident Response Management - Google Cloud Next'18 SF - [Day 1 Keynote](https://youtu.be/vJ9OaAqfxo4?t=2416) showing GKE On-Prem - - [Day 3 – Keynote](https://youtu.be/JQPOPV_VH5w?t=815) showing Stackdriver APM (Tracing, Code Search, Profiler, Google Cloud Build) + - [Day 3 – Keynote](https://youtu.be/JQPOPV_VH5w?t=815) showing Stackdriver + APM (Tracing, Code Search, Profiler, Google Cloud Build) - [Introduction to Service Management with Istio](https://www.youtube.com/watch?v=wCJrdKdD6UM&feature=youtu.be&t=586) --- From 54f0a8d058663a1a9b6060bcdd744eecd2259963 Mon Sep 17 00:00:00 2001 From: rghetia Date: Tue, 22 Jan 2019 09:56:39 -0800 Subject: [PATCH 14/23] add jaeger support for productcatalog and shipping. (#126) --- .../productcatalogservice.yaml | 3 + kubernetes-manifests/shippingservice.yaml | 3 + src/frontend/main.go | 22 +-- src/productcatalogservice/Gopkg.lock | 148 ++++++++++++++---- src/productcatalogservice/server.go | 30 +++- src/shippingservice/Gopkg.lock | 141 +++++++++++++---- src/shippingservice/main.go | 30 +++- 7 files changed, 299 insertions(+), 78 deletions(-) diff --git a/kubernetes-manifests/productcatalogservice.yaml b/kubernetes-manifests/productcatalogservice.yaml index f6a15cd..54416d6 100644 --- a/kubernetes-manifests/productcatalogservice.yaml +++ b/kubernetes-manifests/productcatalogservice.yaml @@ -34,6 +34,9 @@ spec: livenessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:3550"] +# env: +# - name: JAEGER_SERVICE_ADDR +# value: "jaeger-collector:14268" resources: requests: cpu: 100m diff --git a/kubernetes-manifests/shippingservice.yaml b/kubernetes-manifests/shippingservice.yaml index f0b0dae..9213e58 100644 --- a/kubernetes-manifests/shippingservice.yaml +++ b/kubernetes-manifests/shippingservice.yaml @@ -34,6 +34,9 @@ spec: livenessProbe: exec: command: ["/bin/grpc_health_probe", "-addr=:50051"] +# env: +# - name: JAEGER_SERVICE_ADDR +# value: "jaeger-collector:14268" resources: requests: cpu: 100m diff --git a/src/frontend/main.go b/src/frontend/main.go index 61ac8ce..9a19f2e 100644 --- a/src/frontend/main.go +++ b/src/frontend/main.go @@ -181,15 +181,7 @@ func initStats(log logrus.FieldLogger, exporter *stackdriver.Exporter) { } } -func initTracing(log logrus.FieldLogger) { - // This is a demo app with low QPS. trace.AlwaysSample() is used here - // to make sure traces are available for observation and analysis. - // In a production environment or high QPS setup please use - // trace.ProbabilitySampler set at the desired probability. - trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) - - initJaegerTracing(log) - +func initStackDriverTracing(log logrus.FieldLogger) { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. for i := 1; i <= 3; i++ { @@ -215,6 +207,18 @@ func initTracing(log logrus.FieldLogger) { log.Warn("could not initialize stackdriver exporter after retrying, giving up") } +func initTracing(log logrus.FieldLogger) { + // This is a demo app with low QPS. trace.AlwaysSample() is used here + // to make sure traces are available for observation and analysis. + // In a production environment or high QPS setup please use + // trace.ProbabilitySampler set at the desired probability. + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + + initJaegerTracing(log) + initStackDriverTracing(log) + +} + func initProfiling(log logrus.FieldLogger, service, version string) { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. diff --git a/src/productcatalogservice/Gopkg.lock b/src/productcatalogservice/Gopkg.lock index e1c01e0..9e3a02b 100644 --- a/src/productcatalogservice/Gopkg.lock +++ b/src/productcatalogservice/Gopkg.lock @@ -2,27 +2,46 @@ [[projects]] + digest = "1:467af0aad47996b25b838d6f14c8371123a8a76ec239020a6c5894e1f8f60272" name = "cloud.google.com/go" packages = [ "compute/metadata", "internal/version", "monitoring/apiv3", "profiler", - "trace/apiv2" + "trace/apiv2", ] + pruneopts = "UT" revision = "c728a003b238b26cef9ab6753a5dc424b331c3ad" version = "v0.27.0" [[projects]] + digest = "1:4b96dcd8534bc6450a922bd16a76360ba3381f0d1daf40abbaec91c053fbfeb5" name = "contrib.go.opencensus.io/exporter/stackdriver" - packages = [ - ".", - "propagation" - ] + packages = ["."] + pruneopts = "UT" revision = "37aa2801fbf0205003e15636096ebf0373510288" version = "v0.5.0" [[projects]] + branch = "master" + digest = "1:d3a57cdbaefaceca4ebe6258ed86a992bdcfc93a8442dbda5343e2d43a8f8a6a" + name = "git.apache.org/thrift.git" + packages = ["lib/go/thrift"] + pruneopts = "UT" + revision = "67df34afa782be67154034b31e4ad7cb3834fed1" + source = "github.com/apache/thrift" + +[[projects]] + branch = "master" + digest = "1:14e66208d324c0ecb49934b5ac311c50a94e3a458e92b0026ef9e26919ac8d9d" + name = "github.com/GoogleCloudPlatform/microservices-demo" + packages = ["src/productcatalogservice/genproto"] + pruneopts = "UT" + revision = "10dfd04ab174cc680ed6ffef26cc4fb09ec40404" + +[[projects]] + digest = "1:4fbf68bee2a60f6af6414572936edb295f6f26b73c6fb25ab0e7b03b013854f5" name = "github.com/golang/protobuf" packages = [ "jsonpb", @@ -34,46 +53,67 @@ "ptypes/empty", "ptypes/struct", "ptypes/timestamp", - "ptypes/wrappers" + "ptypes/wrappers", ] + pruneopts = "UT" revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" [[projects]] + digest = "1:2e3c336fc7fde5c984d2841455a658a6d626450b1754a854b3b32e7a8f49a07a" name = "github.com/google/go-cmp" packages = [ "cmp", "cmp/internal/diff", "cmp/internal/function", - "cmp/internal/value" + "cmp/internal/value", ] + pruneopts = "UT" revision = "3af367b6b30c263d47e8895973edcca9a49cf029" version = "v0.2.0" [[projects]] branch = "master" + digest = "1:fc55304e290027108ae0cac675a171bcd854f9c657678c20ceea837718ea6819" name = "github.com/google/pprof" packages = ["profile"] - revision = "e027b505a088ac3c68c339a1d7ce7724bf34538b" + pruneopts = "UT" + revision = "e84dfd68c163c45ea47aa24b3dc7eaa93f6675b1" [[projects]] + digest = "1:cd9864c6366515827a759931746738ede6079faa08df9c584596370d6add135c" name = "github.com/googleapis/gax-go" - packages = ["."] - revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f" - version = "v2.0.0" + packages = [ + ".", + "v2", + ] + pruneopts = "UT" + revision = "c8a15bac9b9fe955bd9f900272f9a306465d28cf" + version = "v2.0.3" [[projects]] - digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" + digest = "1:0a69a1c0db3591fcefb47f115b224592c8dfa4368b7ba9fae509d5e16cdc95c8" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "UT" + revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242" + version = "v1.0.1" + +[[projects]] + digest = "1:87c2e02fb01c27060ccc5ba7c5a407cc91147726f8f40b70cceeedbc52b1f3a8" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" - revision = "3e01752db0189b9157070a0e1668a620f9a85da2" - version = "v1.0.6" + revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5" + version = "v1.3.0" [[projects]] + digest = "1:a5154dfd6b37bef5a3eab759e13296348e639dc8c7604f538368167782b08ccd" name = "go.opencensus.io" packages = [ ".", + "exporter/jaeger", + "exporter/jaeger/internal/gen-go/jaeger", "internal", "internal/tagencoding", "plugin/ocgrpc", @@ -86,21 +126,23 @@ "trace", "trace/internal", "trace/propagation", - "trace/tracestate" + "trace/tracestate", ] + pruneopts = "UT" revision = "b11f239c032624b045c4c2bfd3d1287b4012ce89" version = "v0.16.0" [[projects]] branch = "master" - digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" + digest = "1:38f553aff0273ad6f367cb0a0f8b6eecbaef8dc6cb8b50e57b6a81c1d5b1e332" name = "golang.org/x/crypto" packages = ["ssh/terminal"] pruneopts = "UT" - revision = "0e37d006457bf46f9e6692014ba72ef82c33022c" + revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" [[projects]] branch = "master" + digest = "1:9d2f08c64693fbe7177b5980f80c35672c80f12be79bb3bc86948b934d70e4ee" name = "golang.org/x/net" packages = [ "context", @@ -110,35 +152,46 @@ "http2/hpack", "idna", "internal/timeseries", - "trace" + "trace", ] - revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2" + pruneopts = "UT" + revision = "ed066c81e75eba56dd9bd2139ade88125b855585" [[projects]] branch = "master" + digest = "1:511a6232760c10dcb1ebf1ab83ef0291e2baf801f203ca6314759c5458b73a6a" name = "golang.org/x/oauth2" packages = [ ".", "google", "internal", "jws", - "jwt" + "jwt", ] - revision = "d2e6202438beef2727060aa7cabdd924d92ebfd9" + pruneopts = "UT" + revision = "5dab4167f31cbd76b407f1486c86b40748bc5073" [[projects]] branch = "master" + digest = "1:75515eedc0dc2cb0b40372008b616fa2841d831c63eedd403285ff286c593295" name = "golang.org/x/sync" packages = ["semaphore"] - revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + pruneopts = "UT" + revision = "37e7f081c4d4c64e13b10787722085407fe5d15f" [[projects]] branch = "master" + digest = "1:43cde116ff48f299eddb7e6515677e6d0a2c915854bb05a333877f07c3bb3033" name = "golang.org/x/sys" - packages = ["unix"] - revision = "1561086e645b2809fb9f8a1e2a38160bf8d53bf4" + packages = [ + "unix", + "windows", + ] + pruneopts = "UT" + revision = "11f53e03133963fb11ae0588e08b5e0b85be8be5" [[projects]] + digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" name = "golang.org/x/text" packages = [ "collate", @@ -154,13 +207,15 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:26a71f62c83707b9952821c2a895bd041588501fa370cc267221817fcc721253" name = "google.golang.org/api" packages = [ "googleapi/transport", @@ -170,11 +225,14 @@ "support/bundler", "transport", "transport/grpc", - "transport/http" + "transport/http", + "transport/http/internal/propagation", ] - revision = "19ff8768a5c0b8e46ea281065664787eefc24121" + pruneopts = "UT" + revision = "43037ff31f6958582e5d3c19d9ac1a4d2819669c" [[projects]] + digest = "1:c4eaa5f79d36f76ef4bd0c4f96e36bc1b7b5a359528d1267f0cb7a5d58b7b5bb" name = "google.golang.org/appengine" packages = [ ".", @@ -188,13 +246,15 @@ "internal/socket", "internal/urlfetch", "socket", - "urlfetch" + "urlfetch", ] - revision = "ae0ab99deb4dc413a2b4bd6c8bdd0eb67f1e4d06" - version = "v1.2.0" + pruneopts = "UT" + revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" + version = "v1.4.0" [[projects]] branch = "master" + digest = "1:3552e7267a98c605e6cbfe6b03c34589265ab72e6078b95fb2e10e0cb44f5cc8" name = "google.golang.org/genproto" packages = [ "googleapis/api/annotations", @@ -207,11 +267,13 @@ "googleapis/monitoring/v3", "googleapis/rpc/errdetails", "googleapis/rpc/status", - "protobuf/field_mask" + "protobuf/field_mask", ] - revision = "c3f76f3b92d1ffa4c58a9ff842a58b8877655e0f" + pruneopts = "UT" + revision = "db91494dd46c1fdcbbde05e5ff5eb56df8f7d79a" [[projects]] + digest = "1:6497ab07ec89179db8d5a563d33635be04ceffaa29007a3ae74b9f15f4d3068e" name = "google.golang.org/grpc" packages = [ ".", @@ -241,14 +303,32 @@ "resolver/passthrough", "stats", "status", - "tap" + "tap", ] + pruneopts = "UT" revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455" version = "v1.14.0" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b68fd9438c2eb711d9fc51c1f23c5ca0d5169bf4022351dfc400cd35ba39dfaa" + input-imports = [ + "cloud.google.com/go/profiler", + "contrib.go.opencensus.io/exporter/stackdriver", + "github.com/GoogleCloudPlatform/microservices-demo/src/productcatalogservice/genproto", + "github.com/golang/protobuf/jsonpb", + "github.com/golang/protobuf/proto", + "github.com/google/go-cmp/cmp", + "github.com/sirupsen/logrus", + "go.opencensus.io/exporter/jaeger", + "go.opencensus.io/plugin/ocgrpc", + "go.opencensus.io/stats/view", + "go.opencensus.io/trace", + "golang.org/x/net/context", + "google.golang.org/grpc", + "google.golang.org/grpc/codes", + "google.golang.org/grpc/health/grpc_health_v1", + "google.golang.org/grpc/status", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/src/productcatalogservice/server.go b/src/productcatalogservice/server.go index 0fc929f..4fb7018 100644 --- a/src/productcatalogservice/server.go +++ b/src/productcatalogservice/server.go @@ -35,6 +35,7 @@ import ( "contrib.go.opencensus.io/exporter/stackdriver" "github.com/golang/protobuf/jsonpb" "github.com/sirupsen/logrus" + "go.opencensus.io/exporter/jaeger" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/stats/view" "go.opencensus.io/trace" @@ -110,6 +111,28 @@ func run(port int) string { return l.Addr().String() } +func initJaegerTracing() { + svcAddr := os.Getenv("JAEGER_SERVICE_ADDR") + if svcAddr == "" { + log.Info("jaeger initialization disabled.") + return + } + + // Register the Jaeger exporter to be able to retrieve + // the collected spans. + exporter, err := jaeger.NewExporter(jaeger.Options{ + Endpoint: fmt.Sprintf("http://%s", svcAddr), + Process: jaeger.Process{ + ServiceName: "productcatalogservice", + }, + }) + if err != nil { + log.Fatal(err) + } + trace.RegisterExporter(exporter) + log.Info("jaeger initialization completed.") +} + func initStats(exporter *stackdriver.Exporter) { view.SetReportingPeriod(60 * time.Second) view.RegisterExporter(exporter) @@ -120,7 +143,7 @@ func initStats(exporter *stackdriver.Exporter) { } } -func initTracing() { +func initStackDriverTracing() { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. for i := 1; i <= 3; i++ { @@ -143,6 +166,11 @@ func initTracing() { log.Warn("could not initialize stackdriver exporter after retrying, giving up") } +func initTracing() { + initJaegerTracing() + initStackDriverTracing() +} + func initProfiling(service, version string) { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. diff --git a/src/shippingservice/Gopkg.lock b/src/shippingservice/Gopkg.lock index 9250ec1..d190836 100644 --- a/src/shippingservice/Gopkg.lock +++ b/src/shippingservice/Gopkg.lock @@ -2,27 +2,46 @@ [[projects]] + digest = "1:467af0aad47996b25b838d6f14c8371123a8a76ec239020a6c5894e1f8f60272" name = "cloud.google.com/go" packages = [ "compute/metadata", "internal/version", "monitoring/apiv3", "profiler", - "trace/apiv2" + "trace/apiv2", ] + pruneopts = "UT" revision = "c728a003b238b26cef9ab6753a5dc424b331c3ad" version = "v0.27.0" [[projects]] + digest = "1:4b96dcd8534bc6450a922bd16a76360ba3381f0d1daf40abbaec91c053fbfeb5" name = "contrib.go.opencensus.io/exporter/stackdriver" - packages = [ - ".", - "propagation" - ] + packages = ["."] + pruneopts = "UT" revision = "37aa2801fbf0205003e15636096ebf0373510288" version = "v0.5.0" [[projects]] + branch = "master" + digest = "1:d3a57cdbaefaceca4ebe6258ed86a992bdcfc93a8442dbda5343e2d43a8f8a6a" + name = "git.apache.org/thrift.git" + packages = ["lib/go/thrift"] + pruneopts = "UT" + revision = "67df34afa782be67154034b31e4ad7cb3834fed1" + source = "github.com/apache/thrift" + +[[projects]] + branch = "master" + digest = "1:27490301253ac5063d502480ef3794b95222eea6cb997ae6e689a058b1cd5253" + name = "github.com/GoogleCloudPlatform/microservices-demo" + packages = ["src/shippingservice/genproto"] + pruneopts = "UT" + revision = "10dfd04ab174cc680ed6ffef26cc4fb09ec40404" + +[[projects]] + digest = "1:72856926f8208767b837bf51e3373f49139f61889b67dc7fd3c2a0fd711e3f7a" name = "github.com/golang/protobuf" packages = [ "proto", @@ -33,35 +52,54 @@ "ptypes/empty", "ptypes/struct", "ptypes/timestamp", - "ptypes/wrappers" + "ptypes/wrappers", ] + pruneopts = "UT" revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" [[projects]] branch = "master" + digest = "1:fc55304e290027108ae0cac675a171bcd854f9c657678c20ceea837718ea6819" name = "github.com/google/pprof" packages = ["profile"] - revision = "e027b505a088ac3c68c339a1d7ce7724bf34538b" + pruneopts = "UT" + revision = "e84dfd68c163c45ea47aa24b3dc7eaa93f6675b1" [[projects]] + digest = "1:cd9864c6366515827a759931746738ede6079faa08df9c584596370d6add135c" name = "github.com/googleapis/gax-go" - packages = ["."] - revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f" - version = "v2.0.0" + packages = [ + ".", + "v2", + ] + pruneopts = "UT" + revision = "c8a15bac9b9fe955bd9f900272f9a306465d28cf" + version = "v2.0.3" [[projects]] - digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" + digest = "1:0a69a1c0db3591fcefb47f115b224592c8dfa4368b7ba9fae509d5e16cdc95c8" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "UT" + revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242" + version = "v1.0.1" + +[[projects]] + digest = "1:87c2e02fb01c27060ccc5ba7c5a407cc91147726f8f40b70cceeedbc52b1f3a8" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" - revision = "3e01752db0189b9157070a0e1668a620f9a85da2" - version = "v1.0.6" + revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5" + version = "v1.3.0" [[projects]] + digest = "1:a5154dfd6b37bef5a3eab759e13296348e639dc8c7604f538368167782b08ccd" name = "go.opencensus.io" packages = [ ".", + "exporter/jaeger", + "exporter/jaeger/internal/gen-go/jaeger", "internal", "internal/tagencoding", "plugin/ocgrpc", @@ -74,21 +112,23 @@ "trace", "trace/internal", "trace/propagation", - "trace/tracestate" + "trace/tracestate", ] + pruneopts = "UT" revision = "b11f239c032624b045c4c2bfd3d1287b4012ce89" version = "v0.16.0" [[projects]] branch = "master" - digest = "1:3f3a05ae0b95893d90b9b3b5afdb79a9b3d96e4e36e099d841ae602e4aca0da8" + digest = "1:38f553aff0273ad6f367cb0a0f8b6eecbaef8dc6cb8b50e57b6a81c1d5b1e332" name = "golang.org/x/crypto" packages = ["ssh/terminal"] pruneopts = "UT" - revision = "0e37d006457bf46f9e6692014ba72ef82c33022c" + revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" [[projects]] branch = "master" + digest = "1:9d2f08c64693fbe7177b5980f80c35672c80f12be79bb3bc86948b934d70e4ee" name = "golang.org/x/net" packages = [ "context", @@ -98,35 +138,46 @@ "http2/hpack", "idna", "internal/timeseries", - "trace" + "trace", ] - revision = "26e67e76b6c3f6ce91f7c52def5af501b4e0f3a2" + pruneopts = "UT" + revision = "ed066c81e75eba56dd9bd2139ade88125b855585" [[projects]] branch = "master" + digest = "1:511a6232760c10dcb1ebf1ab83ef0291e2baf801f203ca6314759c5458b73a6a" name = "golang.org/x/oauth2" packages = [ ".", "google", "internal", "jws", - "jwt" + "jwt", ] - revision = "d2e6202438beef2727060aa7cabdd924d92ebfd9" + pruneopts = "UT" + revision = "5dab4167f31cbd76b407f1486c86b40748bc5073" [[projects]] branch = "master" + digest = "1:75515eedc0dc2cb0b40372008b616fa2841d831c63eedd403285ff286c593295" name = "golang.org/x/sync" packages = ["semaphore"] - revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + pruneopts = "UT" + revision = "37e7f081c4d4c64e13b10787722085407fe5d15f" [[projects]] branch = "master" + digest = "1:43cde116ff48f299eddb7e6515677e6d0a2c915854bb05a333877f07c3bb3033" name = "golang.org/x/sys" - packages = ["unix"] - revision = "1561086e645b2809fb9f8a1e2a38160bf8d53bf4" + packages = [ + "unix", + "windows", + ] + pruneopts = "UT" + revision = "11f53e03133963fb11ae0588e08b5e0b85be8be5" [[projects]] + digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" name = "golang.org/x/text" packages = [ "collate", @@ -142,13 +193,15 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:26a71f62c83707b9952821c2a895bd041588501fa370cc267221817fcc721253" name = "google.golang.org/api" packages = [ "googleapi/transport", @@ -158,11 +211,14 @@ "support/bundler", "transport", "transport/grpc", - "transport/http" + "transport/http", + "transport/http/internal/propagation", ] - revision = "19ff8768a5c0b8e46ea281065664787eefc24121" + pruneopts = "UT" + revision = "43037ff31f6958582e5d3c19d9ac1a4d2819669c" [[projects]] + digest = "1:c4eaa5f79d36f76ef4bd0c4f96e36bc1b7b5a359528d1267f0cb7a5d58b7b5bb" name = "google.golang.org/appengine" packages = [ ".", @@ -176,13 +232,15 @@ "internal/socket", "internal/urlfetch", "socket", - "urlfetch" + "urlfetch", ] - revision = "ae0ab99deb4dc413a2b4bd6c8bdd0eb67f1e4d06" - version = "v1.2.0" + pruneopts = "UT" + revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" + version = "v1.4.0" [[projects]] branch = "master" + digest = "1:3552e7267a98c605e6cbfe6b03c34589265ab72e6078b95fb2e10e0cb44f5cc8" name = "google.golang.org/genproto" packages = [ "googleapis/api/annotations", @@ -195,11 +253,13 @@ "googleapis/monitoring/v3", "googleapis/rpc/errdetails", "googleapis/rpc/status", - "protobuf/field_mask" + "protobuf/field_mask", ] - revision = "c3f76f3b92d1ffa4c58a9ff842a58b8877655e0f" + pruneopts = "UT" + revision = "db91494dd46c1fdcbbde05e5ff5eb56df8f7d79a" [[projects]] + digest = "1:f3fea5ef1fb1f632ae0dd9a86af6aa2048f3243d1da0075706fca1def38d9fbb" name = "google.golang.org/grpc" packages = [ ".", @@ -231,14 +291,29 @@ "resolver/passthrough", "stats", "status", - "tap" + "tap", ] + pruneopts = "UT" revision = "32fb0ac620c32ba40a4626ddf94d90d12cce3455" version = "v1.14.0" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5dac84cd5efcc89491aa3cda06a774c1b370c8e0cbdb99668cbfd8ba27b3e32c" + input-imports = [ + "cloud.google.com/go/profiler", + "contrib.go.opencensus.io/exporter/stackdriver", + "github.com/GoogleCloudPlatform/microservices-demo/src/shippingservice/genproto", + "github.com/golang/protobuf/proto", + "github.com/sirupsen/logrus", + "go.opencensus.io/exporter/jaeger", + "go.opencensus.io/plugin/ocgrpc", + "go.opencensus.io/stats/view", + "go.opencensus.io/trace", + "golang.org/x/net/context", + "google.golang.org/grpc", + "google.golang.org/grpc/health/grpc_health_v1", + "google.golang.org/grpc/reflection", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/src/shippingservice/main.go b/src/shippingservice/main.go index 523731c..ea324fe 100644 --- a/src/shippingservice/main.go +++ b/src/shippingservice/main.go @@ -23,6 +23,7 @@ import ( "cloud.google.com/go/profiler" "contrib.go.opencensus.io/exporter/stackdriver" "github.com/sirupsen/logrus" + "go.opencensus.io/exporter/jaeger" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/stats/view" "go.opencensus.io/trace" @@ -128,6 +129,28 @@ func (s *server) ShipOrder(ctx context.Context, in *pb.ShipOrderRequest) (*pb.Sh }, nil } +func initJaegerTracing() { + svcAddr := os.Getenv("JAEGER_SERVICE_ADDR") + if svcAddr == "" { + log.Info("jaeger initialization disabled.") + return + } + + // Register the Jaeger exporter to be able to retrieve + // the collected spans. + exporter, err := jaeger.NewExporter(jaeger.Options{ + Endpoint: fmt.Sprintf("http://%s", svcAddr), + Process: jaeger.Process{ + ServiceName: "shippingservice", + }, + }) + if err != nil { + log.Fatal(err) + } + trace.RegisterExporter(exporter) + log.Info("jaeger initialization completed.") +} + func initStats(exporter *stackdriver.Exporter) { view.SetReportingPeriod(60 * time.Second) view.RegisterExporter(exporter) @@ -138,7 +161,7 @@ func initStats(exporter *stackdriver.Exporter) { } } -func initTracing() { +func initStackDriverTracing() { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. for i := 1; i <= 3; i++ { @@ -161,6 +184,11 @@ func initTracing() { log.Warn("could not initialize stackdriver exporter after retrying, giving up") } +func initTracing() { + initJaegerTracing() + initStackDriverTracing() +} + func initProfiling(service, version string) { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. From 2ef073f6009783049a5feb8628e0797c7053319b Mon Sep 17 00:00:00 2001 From: sebright Date: Tue, 22 Jan 2019 09:56:50 -0800 Subject: [PATCH 15/23] adservice: Add "time" field to JSON log entries. (#129) The Log4j JsonLayout puts the log entry timestamp in a field named "instant" by default, but the Stackdriver Logging agent does not understand that field. The logging agent instead uses the time that it received the log entry, which is less accurate and has only second-level precision. This commit adds a key-value pair to the JsonLayout pattern that can be understood by the logging agent. It uses a "time" key as described in https://cloud.google.com/logging/docs/agent/configuration#timestamp-processing and formats the timestamp as described in the Protocol Buffer JSON mapping, https://developers.google.com/protocol-buffers/docs/proto3#json. Allowing the Stackdriver Logging agent to read the more accurate timestamps inserted by Log4j is especially important in the adservice, because the logs are correlated with traces, and it is important to see where each message was logged on the timeline of the trace. --- src/adservice/src/main/resources/log4j2.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/adservice/src/main/resources/log4j2.xml b/src/adservice/src/main/resources/log4j2.xml index 177c483..9d85f97 100644 --- a/src/adservice/src/main/resources/log4j2.xml +++ b/src/adservice/src/main/resources/log4j2.xml @@ -3,12 +3,14 @@ - + + From 3812cf741da92bf4c4140f26b8ede047b69e0732 Mon Sep 17 00:00:00 2001 From: rghetia Date: Tue, 22 Jan 2019 10:37:03 -0800 Subject: [PATCH 16/23] fix typo in Stackdriver init method name. (#130) --- src/frontend/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/frontend/main.go b/src/frontend/main.go index 9a19f2e..a6bccda 100644 --- a/src/frontend/main.go +++ b/src/frontend/main.go @@ -181,7 +181,7 @@ func initStats(log logrus.FieldLogger, exporter *stackdriver.Exporter) { } } -func initStackDriverTracing(log logrus.FieldLogger) { +func initStackdriverTracing(log logrus.FieldLogger) { // TODO(ahmetb) this method is duplicated in other microservices using Go // since they are not sharing packages. for i := 1; i <= 3; i++ { @@ -215,7 +215,7 @@ func initTracing(log logrus.FieldLogger) { trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) initJaegerTracing(log) - initStackDriverTracing(log) + initStackdriverTracing(log) } From ea424cb9f7a23f69d0ce1b7793c28ec6660550df Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Tue, 22 Jan 2019 12:30:30 -0800 Subject: [PATCH 17/23] add skaffold version note Fixes #127 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e6d0828..a6476dc 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). - Docker for Desktop (Mac/Windows): It provides Kubernetes support as [noted here](https://docs.docker.com/docker-for-mac/kubernetes/). - [skaffold](https://github.com/GoogleContainerTools/skaffold/#installation) + (ensure version ≥v0.20) 1. Launch “Docker for Desktop”. Go to Preferences: - choose “Enable Kubernetes”, From 1d452f449a56608e72d88372ae47b8dce9e364c8 Mon Sep 17 00:00:00 2001 From: Megan O'Keefe <3137106+m-okeefe@users.noreply.github.com> Date: Tue, 29 Jan 2019 11:23:33 -0800 Subject: [PATCH 18/23] Adds make-release script, static k8s manifests (#132) Closes #75. ### Changelog Adds 4 scripts to the `hack/` directory for building/pushing images, injecting images tags into static manifests, and tagging new releases. See [hack/README.md](https://github.com/m-okeefe/microservices-demo/tree/release-script/hack). **Note**: since we have not pushed images yet, the images in the `./release/` manifests are still set to the skaffold defaults (eg. `adservice`). --- README.md | 17 ++++++++++- hack/README.md | 18 +++++++++++ hack/make-docker-images.sh | 39 ++++++++++++++++++++++++ hack/make-release-artifacts.sh | 55 ++++++++++++++++++++++++++++++++++ hack/make-release.sh | 41 +++++++++++++++++++++++++ release/.googleheader | 13 ++++++++ 6 files changed, 182 insertions(+), 1 deletion(-) create mode 100755 hack/README.md create mode 100755 hack/make-docker-images.sh create mode 100755 hack/make-release-artifacts.sh create mode 100755 hack/make-release.sh create mode 100644 release/.googleheader diff --git a/README.md b/README.md index a6476dc..a603b4c 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,21 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). are seeing this, run `kubectl get service frontend-external -o=yaml | kubectl apply -f-` to trigger load balancer reconfiguration. +### Option 3: Using Static Images + +> 💡 Recommended for test-driving the application on an existing cluster. + +**Prerequisite**: a running Kubernetes cluster. + +1. Clone this repository. +1. Deploy the application: `kubectl apply -f ./release/kubernetes-manifests` +1. Run `kubectl get pods` to see pods are in a healthy and ready state. +1. Find the IP address of your application, then visit the application on your + browser to confirm installation. + + kubectl get service frontend-external + + ### (Optional) Deploying on a Istio-installed GKE cluster > **Note:** you followed GKE deployment steps above, run `skaffold delete` first @@ -205,4 +220,4 @@ Find **Protocol Buffers Descriptions** at the [`./pb` directory](./pb). --- -This is not an official Google project. +This is not an official Google project. \ No newline at end of file diff --git a/hack/README.md b/hack/README.md new file mode 100755 index 0000000..41d89df --- /dev/null +++ b/hack/README.md @@ -0,0 +1,18 @@ +## `hack/` + +This directory provides scripts for building and pushing Docker images, and tagging new demo +releases. + +### env variables + +- `TAG` - git release tag / Docker tag. +- `REPO_PREFIX` - Docker repo prefix to push images. Format: `$user/$project`. Resulting images will be of the + format `$user/$project/$svcname:$tag` (where `svcname` = `adservice`, `cartservice`, + etc.) + +### scripts + +1. `./make-docker-images.sh`: builds and pushes images to the specified Docker repository. +2. `./make-release-artifacts.sh`: generates a combined YAML file with image $TAG at: + `./release/kubernetes-manifests/demo.yaml`. +3. `./make-release.sh`: runs scripts 1 and 2, then runs `git tag` / pushes updated manifests to master. diff --git a/hack/make-docker-images.sh b/hack/make-docker-images.sh new file mode 100755 index 0000000..51a8e10 --- /dev/null +++ b/hack/make-docker-images.sh @@ -0,0 +1,39 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds and pushes docker image for each demo microservice. + +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "$1" >&2; } +fail() { log "$1"; exit 1; } + +TAG="${TAG?TAG env variable must be specified}" +REPO_PREFIX="${REPO_PREFIX?REPO_PREFIX env variable must be specified}" + + +for dir in ./src/*/ +do + # build image + svcname="$(basename $dir)" + image="$REPO_PREFIX/$svcname:$TAG" + echo "Building and pushing $image..." + docker build -t $image -f $dir/Dockerfile $dir + + # push image + docker push $image +done + +log "Successfully built and pushed images." diff --git a/hack/make-release-artifacts.sh b/hack/make-release-artifacts.sh new file mode 100755 index 0000000..1479f8a --- /dev/null +++ b/hack/make-release-artifacts.sh @@ -0,0 +1,55 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# injects new image/tag into the images in ./release/kubernetes-manifests/demo.yaml + +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "$1" >&2; } +fail() { log "$1"; exit 1; } + +TAG="${TAG?TAG env variable must be specified}" +REPO_PREFIX="${REPO_PREFIX?REPO_PREFIX env variable must be specified}" + +# overwrite release/ with the latest manifests, adding "---" separator. +src="./kubernetes-manifests/*" +manifestfile="./release/kubernetes-manifests/demo.yaml" +tmp="./release/kubernetes-manifests/tmp.yaml" +[ -e $manifestfile ] && rm $manifestfile +for f in $src; do (cat "${f}"; echo "---") >> $tmp; done + +# remove extra google headers +gsed -i '/^#/d' $tmp + +# remove empty lines +gsed -r -i '/^\s*$/d' $tmp + +# add 1 google header to the top +cat "./release/.googleheader" $tmp > $manifestfile +rm $tmp + + +# replace image repo, tag for each deployment +for dir in ./src/*/ +do + svcname="$(basename $dir)" + image="$REPO_PREFIX/$svcname:$TAG" + + pattern="^(\s*)image:\s.*$svcname(.*)(\s*)" + replace="\1image: $image\3" + gsed -r -i "s|$pattern|$replace|g" $manifestfile +done + +log "Successfully added image tags > wrote to demo.yaml". diff --git a/hack/make-release.sh b/hack/make-release.sh new file mode 100755 index 0000000..1eb8bce --- /dev/null +++ b/hack/make-release.sh @@ -0,0 +1,41 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Creates a new release by 1) building/pushing images, 2) injecting tag into YAML, +# 3) creating a new git tag, and 4) pushing tags/updated YAML to $BRANCH. + +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "$1" >&2; } +fail() { log "$1"; exit 1; } + +TAG="${TAG?TAG env variable must be specified}" +REPO_PREFIX="${REPO_PREFIX?REPO_PREFIX env variable must be specified}" + +# build and push images +./hack/make-docker-images.sh + +# update yaml +./hack/make-release-artifacts.sh + +# create git release / push to master +log "Pushing k8s manifests to master..." +git tag "$TAG" +git add release/ +git commit --allow-empty -m "Release $TAG" +git push --tags +git push origin master + +log "Successfully tagged release $TAG." diff --git a/release/.googleheader b/release/.googleheader new file mode 100644 index 0000000..ddd71c0 --- /dev/null +++ b/release/.googleheader @@ -0,0 +1,13 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 3da0ae3b3122f35f6e0984e8c2e58ec1ec758e26 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Tue, 29 Jan 2019 14:05:37 -0800 Subject: [PATCH 19/23] hack: polish scripts (#133) Signed-off-by: Ahmet Alp Balkan --- hack/make-docker-images.sh | 32 ++++++++++++----------- hack/make-release-artifacts.sh | 47 +++++++++++++++++----------------- hack/make-release.sh | 11 ++++---- release/.googleheader | 13 ---------- 4 files changed, 47 insertions(+), 56 deletions(-) delete mode 100644 release/.googleheader diff --git a/hack/make-docker-images.sh b/hack/make-docker-images.sh index 51a8e10..cd4cc63 100755 --- a/hack/make-docker-images.sh +++ b/hack/make-docker-images.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,28 +14,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Builds and pushes docker image for each demo microservice. +# Builds and pushes docker image for each demo microservice. -#!/usr/bin/env bash set -euo pipefail +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" log() { echo "$1" >&2; } -fail() { log "$1"; exit 1; } TAG="${TAG?TAG env variable must be specified}" REPO_PREFIX="${REPO_PREFIX?REPO_PREFIX env variable must be specified}" +while IFS= read -d $'\0' -r dir; do + # build image + svcname="$(basename "${dir}")" + image="${REPO_PREFIX}/$svcname:$TAG" + ( + cd "${dir}" + log "Building: ${image}" + docker build -t "${image}" . -for dir in ./src/*/ -do - # build image - svcname="$(basename $dir)" - image="$REPO_PREFIX/$svcname:$TAG" - echo "Building and pushing $image..." - docker build -t $image -f $dir/Dockerfile $dir + log "Pushing: ${image}" + docker push "${image}" + ) +done < <(find "${SCRIPTDIR}/../src" -mindepth 1 -maxdepth 1 -type d -print0) - # push image - docker push $image -done - -log "Successfully built and pushed images." +log "Successfully built and pushed all images." diff --git a/hack/make-release-artifacts.sh b/hack/make-release-artifacts.sh index 1479f8a..aacb4b1 100755 --- a/hack/make-release-artifacts.sh +++ b/hack/make-release-artifacts.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,42 +16,41 @@ # injects new image/tag into the images in ./release/kubernetes-manifests/demo.yaml -#!/usr/bin/env bash set -euo pipefail +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" log() { echo "$1" >&2; } -fail() { log "$1"; exit 1; } TAG="${TAG?TAG env variable must be specified}" REPO_PREFIX="${REPO_PREFIX?REPO_PREFIX env variable must be specified}" +out_file="${SCRIPTDIR}/../release/kubernetes-manifests/demo.yaml" -# overwrite release/ with the latest manifests, adding "---" separator. -src="./kubernetes-manifests/*" -manifestfile="./release/kubernetes-manifests/demo.yaml" -tmp="./release/kubernetes-manifests/tmp.yaml" -[ -e $manifestfile ] && rm $manifestfile -for f in $src; do (cat "${f}"; echo "---") >> $tmp; done +read_manifests() { + local src_manifest_dir + src_manifest_dir="${SCRIPTDIR}/../kubernetes-manifests" -# remove extra google headers -gsed -i '/^#/d' $tmp + while IFS= read -d $'\0' -r file; do + cat "${file}" + echo "---" + done < <(find "${src_manifest_dir}" -name '*.yaml' -type f -print0) +} -# remove empty lines -gsed -r -i '/^\s*$/d' $tmp +# read and merge all manifests +out_manifest="$(read_manifests)" -# add 1 google header to the top -cat "./release/.googleheader" $tmp > $manifestfile -rm $tmp - - -# replace image repo, tag for each deployment -for dir in ./src/*/ +# replace "image" repo, tag for each service +for dir in ./src/*/ do - svcname="$(basename $dir)" + svcname="$(basename "${dir}")" image="$REPO_PREFIX/$svcname:$TAG" pattern="^(\s*)image:\s.*$svcname(.*)(\s*)" - replace="\1image: $image\3" - gsed -r -i "s|$pattern|$replace|g" $manifestfile + replace="\1image: $image\3" + out_manifest="$(gsed -r "s|$pattern|$replace|g" <(echo "${out_manifest}") )" done -log "Successfully added image tags > wrote to demo.yaml". +rm -rf -- "${out_file}" +mkdir -p "$(dirname "${out_file}")" +echo "${out_manifest}" > "${out_file}" + +log "Successfully saved merged manifests to ${out_file}." diff --git a/hack/make-release.sh b/hack/make-release.sh index 1eb8bce..ff49ff4 100755 --- a/hack/make-release.sh +++ b/hack/make-release.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +17,6 @@ # Creates a new release by 1) building/pushing images, 2) injecting tag into YAML, # 3) creating a new git tag, and 4) pushing tags/updated YAML to $BRANCH. -#!/usr/bin/env bash set -euo pipefail log() { echo "$1" >&2; } @@ -24,13 +25,13 @@ fail() { log "$1"; exit 1; } TAG="${TAG?TAG env variable must be specified}" REPO_PREFIX="${REPO_PREFIX?REPO_PREFIX env variable must be specified}" -# build and push images +# build and push images ./hack/make-docker-images.sh -# update yaml -./hack/make-release-artifacts.sh +# update yaml +./hack/make-release-artifacts.sh -# create git release / push to master +# create git release / push to master log "Pushing k8s manifests to master..." git tag "$TAG" git add release/ diff --git a/release/.googleheader b/release/.googleheader deleted file mode 100644 index ddd71c0..0000000 --- a/release/.googleheader +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. From 1d045de655edf595b037937d9d107d97f498a374 Mon Sep 17 00:00:00 2001 From: Megan O'Keefe <3137106+m-okeefe@users.noreply.github.com> Date: Tue, 29 Jan 2019 16:08:24 -0800 Subject: [PATCH 20/23] productcatalog: introduce configurable latency (#134) --- src/productcatalogservice/README.md | 7 +++++++ src/productcatalogservice/server.go | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/productcatalogservice/README.md b/src/productcatalogservice/README.md index 13ff37c..5dc8fea 100644 --- a/src/productcatalogservice/README.md +++ b/src/productcatalogservice/README.md @@ -29,3 +29,10 @@ kubectl exec \ $(kubectl get pods -l app=productcatalogservice -o jsonpath='{.items[0].metadata.name}') \ -c server -- kill -USR2 1 ``` + +## Latency injection + +This service has an `EXTRA_LATENCY` environment variable. This will inject a sleep for the specified [time.Duration](https://golang.org/pkg/time/#ParseDuration) on every call to +to the server. + +For example, use `EXTRA_LATENCY="5.5s"` to sleep for 5.5 seconds on every request. diff --git a/src/productcatalogservice/server.go b/src/productcatalogservice/server.go index 4fb7018..c55e0af 100644 --- a/src/productcatalogservice/server.go +++ b/src/productcatalogservice/server.go @@ -48,6 +48,7 @@ var ( cat pb.ListProductsResponse catalogMutex *sync.Mutex log *logrus.Logger + extraLatency time.Duration port = flag.Int("port", 3550, "port to listen at") @@ -77,6 +78,18 @@ func main() { go initProfiling("productcatalogservice", "1.0.0") flag.Parse() + // set injected latency + if s := os.Getenv("EXTRA_LATENCY"); s != "" { + v, err := time.ParseDuration(s) + if err != nil { + log.Fatalf("failed to parse EXTRA_LATENCY (%s) as time.Duration: %+v", v, err) + } + extraLatency = v + log.Infof("extra latency enabled (duration: %v)", extraLatency) + } else { + extraLatency = time.Duration(0) + } + sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGUSR1, syscall.SIGUSR2) go func() { @@ -117,7 +130,6 @@ func initJaegerTracing() { log.Info("jaeger initialization disabled.") return } - // Register the Jaeger exporter to be able to retrieve // the collected spans. exporter, err := jaeger.NewExporter(jaeger.Options{ @@ -226,10 +238,12 @@ func (p *productCatalog) Check(ctx context.Context, req *healthpb.HealthCheckReq } func (p *productCatalog) ListProducts(context.Context, *pb.Empty) (*pb.ListProductsResponse, error) { + time.Sleep(extraLatency) return &pb.ListProductsResponse{Products: parseCatalog()}, nil } func (p *productCatalog) GetProduct(ctx context.Context, req *pb.GetProductRequest) (*pb.Product, error) { + time.Sleep(extraLatency) var found *pb.Product for i := 0; i < len(parseCatalog()); i++ { if req.Id == parseCatalog()[i].Id { @@ -243,6 +257,7 @@ func (p *productCatalog) GetProduct(ctx context.Context, req *pb.GetProductReque } func (p *productCatalog) SearchProducts(ctx context.Context, req *pb.SearchProductsRequest) (*pb.SearchProductsResponse, error) { + time.Sleep(extraLatency) // Intepret query as a substring match in name or description. var ps []*pb.Product for _, p := range parseCatalog() { From 02b2018b0ee61f3e0d41094bef3203bdb6ab3eb5 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Wed, 30 Jan 2019 00:24:07 -0800 Subject: [PATCH 21/23] recommendationservice: compile deps from requirements.in (#135) Signed-off-by: Ahmet Alp Balkan --- src/recommendationservice/requirements.in | 6 +++ src/recommendationservice/requirements.txt | 59 ++++++++++++---------- 2 files changed, 38 insertions(+), 27 deletions(-) create mode 100644 src/recommendationservice/requirements.in diff --git a/src/recommendationservice/requirements.in b/src/recommendationservice/requirements.in new file mode 100644 index 0000000..10d437d --- /dev/null +++ b/src/recommendationservice/requirements.in @@ -0,0 +1,6 @@ +google-api-core==1.6.0 +google-python-cloud-debugger==2.9 +grpcio-health-checking==1.13.0 +grpcio==1.16.1 +opencensus[stackdriver]==0.1.10 +python-json-logger==0.1.9 diff --git a/src/recommendationservice/requirements.txt b/src/recommendationservice/requirements.txt index e2dda9b..50114b2 100644 --- a/src/recommendationservice/requirements.txt +++ b/src/recommendationservice/requirements.txt @@ -1,30 +1,35 @@ -cachetools==2.1.0 -certifi==2018.4.16 -chardet==3.0.4 -enum34==1.1.6 -futures==3.2.0 -google-api-core==1.6.0 -google-api-python-client==1.7.4 -google-auth==1.6.1 -google-auth-httplib2==0.0.3 -google-cloud-core==0.29.0 -google-python-cloud-debugger==2.8 -googleapis-common-protos==1.5.3 -grpcio==1.13.0 +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements.txt requirements.in +# +cachetools==3.1.0 # via google-auth +certifi==2018.11.29 # via requests +chardet==3.0.4 # via requests +enum34==1.1.6 # via grpcio +futures==3.2.0 # via google-api-core, grpcio +google-api-core[grpc]==1.6.0 +google-api-python-client==1.7.8 # via google-python-cloud-debugger +google-auth-httplib2==0.0.3 # via google-api-python-client, google-python-cloud-debugger +google-auth==1.6.2 # via google-api-core, google-api-python-client, google-auth-httplib2, google-python-cloud-debugger +google-cloud-core==0.29.1 # via google-cloud-trace +google-cloud-trace==0.20.2 # via opencensus +google-python-cloud-debugger==2.9 +googleapis-common-protos==1.5.6 # via google-api-core grpcio-health-checking==1.13.0 -grpcio-tools==1.0.0 -httplib2==0.11.3 -idna==2.7 +grpcio==1.16.1 +httplib2==0.12.0 # via google-api-python-client, google-auth-httplib2 +idna==2.8 # via requests opencensus[stackdriver]==0.1.10 -protobuf==3.5.2.post1 -pyasn1==0.4.3 -pyasn1-modules==0.2.2 +protobuf==3.6.1 # via google-api-core, googleapis-common-protos, grpcio-health-checking +pyasn1-modules==0.2.4 # via google-auth +pyasn1==0.4.5 # via pyasn1-modules, rsa python-json-logger==0.1.9 -pytz==2018.5 -PyYAML==3.13 -requests==2.20.0 -rsa==3.4.2 -six==1.11.0 -uritemplate==3.0.0 -urllib3==1.23 -virtualenv==16.0.0 +pytz==2018.9 # via google-api-core +pyyaml==3.13 # via google-python-cloud-debugger +requests==2.21.0 # via google-api-core +rsa==4.0 # via google-auth +six==1.12.0 # via google-api-core, google-api-python-client, google-auth, google-python-cloud-debugger, grpcio, protobuf +uritemplate==3.0.0 # via google-api-python-client +urllib3==1.24.1 # via requests From 31df60f050a113c771a91477863ce93c8ed0f078 Mon Sep 17 00:00:00 2001 From: Bogdan Drutu Date: Wed, 30 Jan 2019 13:35:03 -0800 Subject: [PATCH 22/23] Updates deps, OC instrumentation, formatting. (#131) --- src/adservice/build.gradle | 16 +- .../src/main/java/hipstershop/AdService.java | 145 ++++++++++-------- .../java/hipstershop/AdServiceClient.java | 103 ++++++------- 3 files changed, 138 insertions(+), 126 deletions(-) diff --git a/src/adservice/build.gradle b/src/adservice/build.gradle index 7450826..dde9248 100644 --- a/src/adservice/build.gradle +++ b/src/adservice/build.gradle @@ -10,12 +10,14 @@ buildscript { } dependencies { classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3' + classpath "gradle.plugin.com.github.sherter.google-java-format:google-java-format-gradle-plugin:0.7.1" } } apply plugin: 'idea' apply plugin: 'java' apply plugin: 'com.google.protobuf' +apply plugin: 'com.github.sherter.google-java-format' repositories { mavenCentral() @@ -23,10 +25,10 @@ repositories { } group = "adservice" -version = "0.1.0-SNAPSHOT" // CURRENT_OPENCENSUS_VERSION +version = "0.1.0-SNAPSHOT" -def opencensusVersion = "0.17.0" // LATEST_OPENCENSUS_RELEASE_VERSION -def grpcVersion = "1.15.0" // CURRENT_GRPC_VERSION +def opencensusVersion = "0.18.0" +def grpcVersion = "1.17.0" def jacksonVersion = "2.9.6" tasks.withType(JavaCompile) { @@ -43,7 +45,9 @@ dependencies { if (speed) { compile fileTree(dir: offlineCompile, include: '*.jar') } else { - compile "com.google.api.grpc:proto-google-common-protos:1.11.0", + compile "com.google.api.grpc:proto-google-common-protos:1.12.0", + "io.opencensus:opencensus-api:${opencensusVersion}", + "io.opencensus:opencensus-contrib-grpc-util:${opencensusVersion}", "io.opencensus:opencensus-exporter-trace-jaeger:${opencensusVersion}", "io.opencensus:opencensus-exporter-stats-stackdriver:${opencensusVersion}", "io.opencensus:opencensus-exporter-trace-stackdriver:${opencensusVersion}", @@ -79,6 +83,10 @@ protobuf { } } +googleJavaFormat { + toolVersion '1.7' +} + // Inform IDEs like IntelliJ IDEA, Eclipse or NetBeans about the generated code. sourceSets { main { diff --git a/src/adservice/src/main/java/hipstershop/AdService.java b/src/adservice/src/main/java/hipstershop/AdService.java index 7320764..1c04f60 100644 --- a/src/adservice/src/main/java/hipstershop/AdService.java +++ b/src/adservice/src/main/java/hipstershop/AdService.java @@ -16,8 +16,8 @@ package hipstershop; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import hipstershop.Demo.Ad; import hipstershop.Demo.AdRequest; @@ -26,49 +26,50 @@ import io.grpc.Server; import io.grpc.ServerBuilder; import io.grpc.StatusRuntimeException; import io.grpc.health.v1.HealthCheckResponse.ServingStatus; -import io.grpc.stub.StreamObserver; import io.grpc.services.*; +import io.grpc.stub.StreamObserver; import io.opencensus.common.Duration; -import io.opencensus.common.Scope; import io.opencensus.contrib.grpc.metrics.RpcViews; -import io.opencensus.exporter.trace.jaeger.JaegerTraceExporter; -import io.opencensus.exporter.trace.logging.LoggingTraceExporter; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; +import io.opencensus.exporter.trace.jaeger.JaegerTraceExporter; import io.opencensus.exporter.trace.stackdriver.StackdriverTraceConfiguration; import io.opencensus.exporter.trace.stackdriver.StackdriverTraceExporter; import io.opencensus.trace.AttributeValue; import io.opencensus.trace.Span; -import io.opencensus.trace.SpanBuilder; import io.opencensus.trace.Tracer; import io.opencensus.trace.Tracing; -import io.opencensus.trace.samplers.Samplers; import java.io.IOException; -import java.util.Collection; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public final class AdService { -public class AdService { private static final Logger logger = LogManager.getLogger(AdService.class); - private static final Tracer tracer = Tracing.getTracer(); - private int MAX_ADS_TO_SERVE = 2; + private static int MAX_ADS_TO_SERVE = 2; private Server server; private HealthStatusManager healthMgr; - static final AdService service = new AdService(); + private static final AdService service = new AdService(); + private void start() throws IOException { int port = Integer.parseInt(System.getenv("PORT")); healthMgr = new HealthStatusManager(); - server = ServerBuilder.forPort(port).addService(new AdServiceImpl()) - .addService(healthMgr.getHealthService()).build().start(); + server = + ServerBuilder.forPort(port) + .addService(new AdServiceImpl()) + .addService(healthMgr.getHealthService()) + .build() + .start(); logger.info("Ad Service started, listening on " + port); Runtime.getRuntime() .addShutdownHook( @@ -91,26 +92,20 @@ public class AdService { } } - static class AdServiceImpl extends hipstershop.AdServiceGrpc.AdServiceImplBase { + private static class AdServiceImpl extends hipstershop.AdServiceGrpc.AdServiceImplBase { /** * Retrieves ads based on context provided in the request {@code AdRequest}. * * @param req the request containing context. - * @param responseObserver the stream observer which gets notified with the value of - * {@code AdResponse} + * @param responseObserver the stream observer which gets notified with the value of {@code + * AdResponse} */ @Override public void getAds(AdRequest req, StreamObserver responseObserver) { AdService service = AdService.getInstance(); - Span parentSpan = tracer.getCurrentSpan(); - SpanBuilder spanBuilder = - tracer - .spanBuilderWithExplicitParent("Retrieve Ads", parentSpan) - .setRecordEvents(true) - .setSampler(Samplers.alwaysSample()); - try (Scope scope = spanBuilder.startScopedSpan()) { - Span span = tracer.getCurrentSpan(); + Span span = tracer.getCurrentSpan(); + try { span.putAttribute("method", AttributeValue.stringAttributeValue("getAds")); List allAds = new ArrayList<>(); logger.info("received ad request (context_words=" + req.getContextKeysList() + ")"); @@ -140,29 +135,29 @@ public class AdService { responseObserver.onCompleted(); } catch (StatusRuntimeException e) { logger.log(Level.WARN, "GetAds Failed", e.getStatus()); - return; + responseObserver.onError(e); } } } - static final ImmutableListMultimap adsMap = createAdsMap(); + private static final ImmutableListMultimap adsMap = createAdsMap(); - Collection getAdsByCategory(String category) { + private Collection getAdsByCategory(String category) { return adsMap.get(category); } private static final Random random = new Random(); - public List getRandomAds() { + private List getRandomAds() { List ads = new ArrayList<>(MAX_ADS_TO_SERVE); Collection allAds = adsMap.values(); - for (int i=0; i createAdsMap() { - Ad camera = Ad.newBuilder().setRedirectUrl("/product/2ZYFJ3GM2N") - .setText("Film camera for sale. 50% off.").build(); - Ad lens = Ad.newBuilder().setRedirectUrl("/product/66VCHSJNUP") - .setText("Vintage camera lens for sale. 20% off.").build(); - Ad recordPlayer = Ad.newBuilder().setRedirectUrl("/product/0PUK6V6EV0") - .setText("Vintage record player for sale. 30% off.").build(); - Ad bike = Ad.newBuilder().setRedirectUrl("/product/9SIQT8TOJO") - .setText("City Bike for sale. 10% off.").build(); - Ad baristaKit = Ad.newBuilder().setRedirectUrl("/product/1YMWWN1N4O") - .setText("Home Barista kitchen kit for sale. Buy one, get second kit for free").build(); - Ad airPlant = Ad.newBuilder().setRedirectUrl("/product/6E92ZMYYFZ") - .setText("Air plants for sale. Buy two, get third one for free").build(); - Ad terrarium = Ad.newBuilder().setRedirectUrl("/product/L9ECAV7KIM") - .setText("Terrarium for sale. Buy one, get second one for free").build(); + private static ImmutableListMultimap createAdsMap() { + Ad camera = + Ad.newBuilder() + .setRedirectUrl("/product/2ZYFJ3GM2N") + .setText("Film camera for sale. 50% off.") + .build(); + Ad lens = + Ad.newBuilder() + .setRedirectUrl("/product/66VCHSJNUP") + .setText("Vintage camera lens for sale. 20% off.") + .build(); + Ad recordPlayer = + Ad.newBuilder() + .setRedirectUrl("/product/0PUK6V6EV0") + .setText("Vintage record player for sale. 30% off.") + .build(); + Ad bike = + Ad.newBuilder() + .setRedirectUrl("/product/9SIQT8TOJO") + .setText("City Bike for sale. 10% off.") + .build(); + Ad baristaKit = + Ad.newBuilder() + .setRedirectUrl("/product/1YMWWN1N4O") + .setText("Home Barista kitchen kit for sale. Buy one, get second kit for free") + .build(); + Ad airPlant = + Ad.newBuilder() + .setRedirectUrl("/product/6E92ZMYYFZ") + .setText("Air plants for sale. Buy two, get third one for free") + .build(); + Ad terrarium = + Ad.newBuilder() + .setRedirectUrl("/product/L9ECAV7KIM") + .setText("Terrarium for sale. Buy one, get second one for free") + .build(); return ImmutableListMultimap.builder() .putAll("photography", camera, lens) .putAll("vintage", camera, lens, recordPlayer) @@ -197,7 +213,7 @@ public class AdService { .build(); } - public static void initStackdriver() { + private static void initStackdriver() { logger.info("Initialize StackDriver"); long sleepTime = 10; /* seconds */ @@ -205,7 +221,7 @@ public class AdService { boolean statsExporterRegistered = false; boolean traceExporterRegistered = false; - for (int i=0; i Date: Thu, 31 Jan 2019 15:25:33 -0800 Subject: [PATCH 23/23] hack: compile istio manifests as well (#137) - compile k8s manifests to release/kubernetes-manifests.yaml - compile istio manifests to release/istio-manifests.yaml --- hack/make-release-artifacts.sh | 57 ++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/hack/make-release-artifacts.sh b/hack/make-release-artifacts.sh index aacb4b1..213472b 100755 --- a/hack/make-release-artifacts.sh +++ b/hack/make-release-artifacts.sh @@ -14,43 +14,60 @@ # See the License for the specific language governing permissions and # limitations under the License. -# injects new image/tag into the images in ./release/kubernetes-manifests/demo.yaml +# This script compiles manifest files with the image tags and places them in +# /release/... set -euo pipefail SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +[[ -n "${DEBUG:-}" ]] && set -x log() { echo "$1" >&2; } TAG="${TAG?TAG env variable must be specified}" REPO_PREFIX="${REPO_PREFIX?REPO_PREFIX env variable must be specified}" -out_file="${SCRIPTDIR}/../release/kubernetes-manifests/demo.yaml" +OUT_DIR="${OUT_DIR:-${SCRIPTDIR}/../release}" read_manifests() { - local src_manifest_dir - src_manifest_dir="${SCRIPTDIR}/../kubernetes-manifests" + local dir + dir="$1" while IFS= read -d $'\0' -r file; do cat "${file}" echo "---" - done < <(find "${src_manifest_dir}" -name '*.yaml' -type f -print0) + done < <(find "${dir}" -name '*.yaml' -type f -print0) } -# read and merge all manifests -out_manifest="$(read_manifests)" +mk_kubernetes_manifests() { + out_manifest="$(read_manifests "${SCRIPTDIR}/../kubernetes-manifests")" -# replace "image" repo, tag for each service -for dir in ./src/*/ -do - svcname="$(basename "${dir}")" - image="$REPO_PREFIX/$svcname:$TAG" + # replace "image" repo, tag for each service + for dir in ./src/*/ + do + svcname="$(basename "${dir}")" + image="$REPO_PREFIX/$svcname:$TAG" - pattern="^(\s*)image:\s.*$svcname(.*)(\s*)" - replace="\1image: $image\3" - out_manifest="$(gsed -r "s|$pattern|$replace|g" <(echo "${out_manifest}") )" -done + pattern="^(\s*)image:\s.*$svcname(.*)(\s*)" + replace="\1image: $image\3" + out_manifest="$(gsed -r "s|$pattern|$replace|g" <(echo "${out_manifest}") )" + done + echo "${out_manifest}" +} -rm -rf -- "${out_file}" -mkdir -p "$(dirname "${out_file}")" -echo "${out_manifest}" > "${out_file}" +mk_istio_manifests() { + read_manifests "${SCRIPTDIR}/../istio-manifests" +} -log "Successfully saved merged manifests to ${out_file}." +main() { + mkdir -p "${OUT_DIR}" + local k8s_manifests_file istio_manifests_file + + k8s_manifests_file="${OUT_DIR}/kubernetes-manifests.yaml" + mk_kubernetes_manifests > "${k8s_manifests_file}" + log "Written ${k8s_manifests_file}" + + istio_manifests_file="${OUT_DIR}/istio-manifests.yaml" + mk_istio_manifests > "${istio_manifests_file}" + log "Written ${istio_manifests_file}" +} + +main