From 07c669a047b579694d882b0b3ff129f862a63610 Mon Sep 17 00:00:00 2001 From: askmeegs Date: Sun, 18 Aug 2019 15:33:58 -0400 Subject: [PATCH] Increases cartservice health check rpc timeout --- release/kubernetes-manifests.yaml | 7 ++++--- src/cartservice/HealthImpl.cs | 10 +++++++--- src/cartservice/cartservice.csproj | 2 +- src/cartservice/cartstore/RedisCartStore.cs | 21 ++++++++++----------- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/release/kubernetes-manifests.yaml b/release/kubernetes-manifests.yaml index d9827fb..52a3d32 100644 --- a/release/kubernetes-manifests.yaml +++ b/release/kubernetes-manifests.yaml @@ -391,7 +391,8 @@ spec: terminationGracePeriodSeconds: 5 containers: - name: server - image: gcr.io/google-samples/microservices-demo/cartservice:v0.1.1 + image: gcr.io/mokeefe/cartservice:august + imagePullPolicy: Always ports: - containerPort: 7070 env: @@ -411,12 +412,12 @@ spec: readinessProbe: initialDelaySeconds: 15 exec: - command: ["/bin/grpc_health_probe", "-addr=:7070"] + command: ["/bin/grpc_health_probe", "-addr=:7070", "-rpc-timeout=5s"] livenessProbe: initialDelaySeconds: 15 periodSeconds: 10 exec: - command: ["/bin/grpc_health_probe", "-addr=:7070"] + command: ["/bin/grpc_health_probe", "-addr=:7070", "-rpc-timeout=5s"] --- apiVersion: v1 kind: Service diff --git a/src/cartservice/HealthImpl.cs b/src/cartservice/HealthImpl.cs index f231c7c..f535b39 100644 --- a/src/cartservice/HealthImpl.cs +++ b/src/cartservice/HealthImpl.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using static System.Diagnostics.Stopwatch; using cartservice.interfaces; using Grpc.Core; using Grpc.Health.V1; @@ -14,11 +15,14 @@ namespace cartservice { } public override Task Check(HealthCheckRequest request, ServerCallContext context){ - Console.WriteLine ("Checking CartService Health"); - - return Task.FromResult(new HealthCheckResponse { + var watch = StartNew(); + var result = Task.FromResult(new HealthCheckResponse { Status = dependency.Ping() ? HealthCheckResponse.Types.ServingStatus.Serving : HealthCheckResponse.Types.ServingStatus.NotServing }); + watch.Stop(); + var elapsedMs = watch.ElapsedMilliseconds; + Console.WriteLine ("✅ Health Check took " + elapsedMs + "ms"); + return result; } } } diff --git a/src/cartservice/cartservice.csproj b/src/cartservice/cartservice.csproj index 3fea4a1..4c1e378 100644 --- a/src/cartservice/cartservice.csproj +++ b/src/cartservice/cartservice.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/cartservice/cartstore/RedisCartStore.cs b/src/cartservice/cartstore/RedisCartStore.cs index bc7b7e8..30dc955 100644 --- a/src/cartservice/cartstore/RedisCartStore.cs +++ b/src/cartservice/cartstore/RedisCartStore.cs @@ -45,13 +45,13 @@ namespace cartservice.cartstore var cart = new Hipstershop.Cart(); emptyCartBytes = cart.ToByteArray(); connectionString = $"{redisAddress},ssl=false,allowAdmin=true,connectRetry=5"; - + redisConnectionOptions = ConfigurationOptions.Parse(connectionString); // Try to reconnect if first retry failed (up to 5 times with exponential backoff) redisConnectionOptions.ConnectRetry = REDIS_RETRY_NUM; redisConnectionOptions.ReconnectRetryPolicy = new ExponentialRetry(100); - + redisConnectionOptions.KeepAlive = 180; } @@ -78,11 +78,11 @@ namespace cartservice.cartstore Console.WriteLine("Connecting to Redis: " + connectionString); redis = ConnectionMultiplexer.Connect(redisConnectionOptions); - + if (redis == null || !redis.IsConnected) { Console.WriteLine("Wasn't able to connect to redis"); - + // We weren't able to connect to redis despite 5 retries with exponential backoff throw new ApplicationException("Wasn't able to connect to redis"); } @@ -96,14 +96,14 @@ namespace cartservice.cartstore Console.WriteLine($"Small test result: {res}"); redis.InternalError += (o, e) => { Console.WriteLine(e.Exception); }; - redis.ConnectionRestored += (o, e) => + redis.ConnectionRestored += (o, e) => { isRedisConnectionOpened = true; - Console.WriteLine("Connection to redis was retored successfully"); + Console.WriteLine("Connection to redis was retored successfully"); }; - redis.ConnectionFailed += (o, e) => + redis.ConnectionFailed += (o, e) => { - Console.WriteLine("Connection failed. Disposing the object"); + Console.WriteLine("Connection failed. Disposing the object"); isRedisConnectionOpened = false; }; @@ -118,9 +118,9 @@ namespace cartservice.cartstore try { EnsureRedisConnected(); - + var db = redis.GetDatabase(); - + // Access the cart from the cache var value = await db.HashGetAsync(userId, CART_FIELD_NAME); @@ -202,7 +202,6 @@ namespace cartservice.cartstore { try { - var redis = ConnectionMultiplexer.Connect(redisConnectionOptions); var cache = redis.GetDatabase(); var res = cache.Ping(); return res != TimeSpan.Zero;