Increases cartservice health check rpc timeout

This commit is contained in:
askmeegs 2019-08-18 15:33:58 -04:00
parent 13b73067c4
commit 07c669a047
4 changed files with 22 additions and 18 deletions

View file

@ -391,7 +391,8 @@ spec:
terminationGracePeriodSeconds: 5 terminationGracePeriodSeconds: 5
containers: containers:
- name: server - name: server
image: gcr.io/google-samples/microservices-demo/cartservice:v0.1.1 image: gcr.io/mokeefe/cartservice:august
imagePullPolicy: Always
ports: ports:
- containerPort: 7070 - containerPort: 7070
env: env:
@ -411,12 +412,12 @@ spec:
readinessProbe: readinessProbe:
initialDelaySeconds: 15 initialDelaySeconds: 15
exec: exec:
command: ["/bin/grpc_health_probe", "-addr=:7070"] command: ["/bin/grpc_health_probe", "-addr=:7070", "-rpc-timeout=5s"]
livenessProbe: livenessProbe:
initialDelaySeconds: 15 initialDelaySeconds: 15
periodSeconds: 10 periodSeconds: 10
exec: exec:
command: ["/bin/grpc_health_probe", "-addr=:7070"] command: ["/bin/grpc_health_probe", "-addr=:7070", "-rpc-timeout=5s"]
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service

View file

@ -1,5 +1,6 @@
using System; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
using static System.Diagnostics.Stopwatch;
using cartservice.interfaces; using cartservice.interfaces;
using Grpc.Core; using Grpc.Core;
using Grpc.Health.V1; using Grpc.Health.V1;
@ -14,11 +15,14 @@ namespace cartservice {
} }
public override Task<HealthCheckResponse> Check(HealthCheckRequest request, ServerCallContext context){ public override Task<HealthCheckResponse> Check(HealthCheckRequest request, ServerCallContext context){
Console.WriteLine ("Checking CartService Health"); var watch = StartNew();
var result = Task.FromResult(new HealthCheckResponse {
return Task.FromResult(new HealthCheckResponse {
Status = dependency.Ping() ? HealthCheckResponse.Types.ServingStatus.Serving : HealthCheckResponse.Types.ServingStatus.NotServing Status = dependency.Ping() ? HealthCheckResponse.Types.ServingStatus.Serving : HealthCheckResponse.Types.ServingStatus.NotServing
}); });
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
Console.WriteLine ("✅ Health Check took " + elapsedMs + "ms");
return result;
} }
} }
} }

View file

@ -14,7 +14,7 @@
<PackageReference Include="grpc.tools" Version="1.12.0" /> <PackageReference Include="grpc.tools" Version="1.12.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" /> <PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" /> <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" />
<PackageReference Include="StackExchange.Redis" Version="1.2.6" /> <PackageReference Include="StackExchange.Redis" Version="2.0.601" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

View file

@ -45,13 +45,13 @@ namespace cartservice.cartstore
var cart = new Hipstershop.Cart(); var cart = new Hipstershop.Cart();
emptyCartBytes = cart.ToByteArray(); emptyCartBytes = cart.ToByteArray();
connectionString = $"{redisAddress},ssl=false,allowAdmin=true,connectRetry=5"; connectionString = $"{redisAddress},ssl=false,allowAdmin=true,connectRetry=5";
redisConnectionOptions = ConfigurationOptions.Parse(connectionString); redisConnectionOptions = ConfigurationOptions.Parse(connectionString);
// Try to reconnect if first retry failed (up to 5 times with exponential backoff) // Try to reconnect if first retry failed (up to 5 times with exponential backoff)
redisConnectionOptions.ConnectRetry = REDIS_RETRY_NUM; redisConnectionOptions.ConnectRetry = REDIS_RETRY_NUM;
redisConnectionOptions.ReconnectRetryPolicy = new ExponentialRetry(100); redisConnectionOptions.ReconnectRetryPolicy = new ExponentialRetry(100);
redisConnectionOptions.KeepAlive = 180; redisConnectionOptions.KeepAlive = 180;
} }
@ -78,11 +78,11 @@ namespace cartservice.cartstore
Console.WriteLine("Connecting to Redis: " + connectionString); Console.WriteLine("Connecting to Redis: " + connectionString);
redis = ConnectionMultiplexer.Connect(redisConnectionOptions); redis = ConnectionMultiplexer.Connect(redisConnectionOptions);
if (redis == null || !redis.IsConnected) if (redis == null || !redis.IsConnected)
{ {
Console.WriteLine("Wasn't able to connect to redis"); Console.WriteLine("Wasn't able to connect to redis");
// We weren't able to connect to redis despite 5 retries with exponential backoff // We weren't able to connect to redis despite 5 retries with exponential backoff
throw new ApplicationException("Wasn't able to connect to redis"); throw new ApplicationException("Wasn't able to connect to redis");
} }
@ -96,14 +96,14 @@ namespace cartservice.cartstore
Console.WriteLine($"Small test result: {res}"); Console.WriteLine($"Small test result: {res}");
redis.InternalError += (o, e) => { Console.WriteLine(e.Exception); }; redis.InternalError += (o, e) => { Console.WriteLine(e.Exception); };
redis.ConnectionRestored += (o, e) => redis.ConnectionRestored += (o, e) =>
{ {
isRedisConnectionOpened = true; isRedisConnectionOpened = true;
Console.WriteLine("Connection to redis was retored successfully"); Console.WriteLine("Connection to redis was retored successfully");
}; };
redis.ConnectionFailed += (o, e) => redis.ConnectionFailed += (o, e) =>
{ {
Console.WriteLine("Connection failed. Disposing the object"); Console.WriteLine("Connection failed. Disposing the object");
isRedisConnectionOpened = false; isRedisConnectionOpened = false;
}; };
@ -118,9 +118,9 @@ namespace cartservice.cartstore
try try
{ {
EnsureRedisConnected(); EnsureRedisConnected();
var db = redis.GetDatabase(); var db = redis.GetDatabase();
// Access the cart from the cache // Access the cart from the cache
var value = await db.HashGetAsync(userId, CART_FIELD_NAME); var value = await db.HashGetAsync(userId, CART_FIELD_NAME);
@ -202,7 +202,6 @@ namespace cartservice.cartstore
{ {
try try
{ {
var redis = ConnectionMultiplexer.Connect(redisConnectionOptions);
var cache = redis.GetDatabase(); var cache = redis.GetDatabase();
var res = cache.Ping(); var res = cache.Ping();
return res != TimeSpan.Zero; return res != TimeSpan.Zero;