vbatts/microservices-demo
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Re-enable opencensus python (#103)

Browse source 
Enables tracing in the email and recommendation services, which was disabled in 316db88 because of a memory leak in the stackdriver exporter.

We fixed the leak in https://github.com/googleapis/google-cloud-python/pull/6856. The fix is included in the [0.1.10 release of opencensus-python](https://github.com/census-instrumentation/opencensus-python/releases/tag/v0.1.10).

With this diff, traces show up as expected in stackdriver while running the demo on GKE. Using an `opencensus-python` package version before `0.1.10` causes the email and recommendation services to leak memory until they OOM. Memory use is back to normal (i.e. roughly constant) using the new package version.
This commit is contained in:
Chris Kleinknecht 2018-12-11 16:15:51 -08:00 committed by Ahmet Alp Balkan
parent cb241e197d
commit 5272a4d821
6 changed files with 47 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/emailservice/Dockerfile
View file

@ -13,7 +13,7 @@ RUN apk add --update --no-cache \
# App Deps # App Deps
cairo-dev \ cairo-dev \
cairo \ cairo \
openssl-dev \ openssl-dev \
gobject-introspection-dev gobject-introspection-dev
# get packages # get packages
@ -42,4 +42,4 @@ RUN apk add --no-cache libstdc++
COPY . . COPY . .
EXPOSE 8080 EXPOSE 8080
ENTRYPOINT [ "python", "email_server.py" ] ENTRYPOINT [ "python", "email_server.py" ]

20
src/emailservice/email_client.py
View file

@ -22,20 +22,20 @@ import demo_pb2_grpc
from logger import getJSONLogger from logger import getJSONLogger
logger = getJSONLogger('emailservice-client') logger = getJSONLogger('emailservice-client')
# from opencensus.trace.tracer import Tracer from opencensus.trace.tracer import Tracer
# from opencensus.trace.exporters import stackdriver_exporter from opencensus.trace.exporters import stackdriver_exporter
# from opencensus.trace.ext.grpc import client_interceptor from opencensus.trace.ext.grpc import client_interceptor
# try: try:
# exporter = stackdriver_exporter.StackdriverExporter() exporter = stackdriver_exporter.StackdriverExporter()
# tracer = Tracer(exporter=exporter) tracer = Tracer(exporter=exporter)
# tracer_interceptor = client_interceptor.OpenCensusClientInterceptor(tracer, host_port='0.0.0.0:8080') tracer_interceptor = client_interceptor.OpenCensusClientInterceptor(tracer, host_port='0.0.0.0:8080')
# except: except:
# tracer_interceptor = client_interceptor.OpenCensusClientInterceptor() tracer_interceptor = client_interceptor.OpenCensusClientInterceptor()
def send_confirmation_email(email, order): def send_confirmation_email(email, order):
channel = grpc.insecure_channel('0.0.0.0:8080') channel = grpc.insecure_channel('0.0.0.0:8080')
# channel = grpc.intercept_channel(channel, tracer_interceptor) channel = grpc.intercept_channel(channel, tracer_interceptor)
stub = demo_pb2_grpc.EmailServiceStub(channel) stub = demo_pb2_grpc.EmailServiceStub(channel)
try: try:
response = stub.SendOrderConfirmation(demo_pb2.SendOrderConfirmationRequest( response = stub.SendOrderConfirmation(demo_pb2.SendOrderConfirmationRequest(

22
src/emailservice/email_server.py
View file

@ -28,19 +28,18 @@ import demo_pb2_grpc
from grpc_health.v1 import health_pb2 from grpc_health.v1 import health_pb2
from grpc_health.v1 import health_pb2_grpc from grpc_health.v1 import health_pb2_grpc
# from opencensus.trace.ext.grpc import server_interceptor from opencensus.trace.exporters import stackdriver_exporter
# from opencensus.trace.samplers import always_on from opencensus.trace.ext.grpc import server_interceptor
# from opencensus.trace.exporters import stackdriver_exporter from opencensus.trace.samplers import always_on
# from opencensus.trace.exporters import print_exporter
# import googleclouddebugger # import googleclouddebugger
# try: try:
# sampler = always_on.AlwaysOnSampler() sampler = always_on.AlwaysOnSampler()
# exporter = stackdriver_exporter.StackdriverExporter() exporter = stackdriver_exporter.StackdriverExporter()
# tracer_interceptor = server_interceptor.OpenCensusServerInterceptor(sampler, exporter) tracer_interceptor = server_interceptor.OpenCensusServerInterceptor(sampler, exporter)
# except: except:
# tracer_interceptor = server_interceptor.OpenCensusServerInterceptor() tracer_interceptor = server_interceptor.OpenCensusServerInterceptor()
# try: # try:
# googleclouddebugger.enable( # googleclouddebugger.enable(
@ -123,7 +122,8 @@ class HealthCheck():
status=health_pb2.HealthCheckResponse.SERVING) status=health_pb2.HealthCheckResponse.SERVING)
def start(dummy_mode): def start(dummy_mode):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))#, interceptors=(tracer_interceptor,)) server = grpc.server(futures.ThreadPoolExecutor(max_workers=10),
interceptors=(tracer_interceptor,))
service = None service = None
if dummy_mode: if dummy_mode:
service = DummyEmailService() service = DummyEmailService()

11
src/emailservice/requirements.txt
View file

@ -8,13 +8,12 @@ cryptography==2.3.1
entrypoints==0.2.3 entrypoints==0.2.3
enum34==1.1.6 enum34==1.1.6
futures==3.1.1 futures==3.1.1
google-api-core==1.4.0 google-api-core==1.6.0
google-auth==1.5.1 google-auth==1.6.1
google-cloud-core==0.28.1 google-cloud-core==0.29.0
google-cloud-trace==0.19.0
googleapis-common-protos==1.5.3 googleapis-common-protos==1.5.3
grpc-google-iam-v1==0.11.4 grpc-google-iam-v1==0.11.4
grpcio==1.12.1 grpcio==1.16.1
grpcio-health-checking==1.12.1 grpcio-health-checking==1.12.1
grpcio-tools==1.12.1 grpcio-tools==1.12.1
idna==2.7 idna==2.7
@ -24,7 +23,7 @@ Jinja2==2.10
keyring==15.1.0 keyring==15.1.0
keyrings.alt==3.1 keyrings.alt==3.1
MarkupSafe==1.0 MarkupSafe==1.0
opencensus==0.1.7 opencensus[stackdriver]==0.1.10
protobuf==3.6.1 protobuf==3.6.1
pyasn1==0.4.4 pyasn1==0.4.4
pyasn1-modules==0.2.2 pyasn1-modules==0.2.2

33
src/recommendationservice/recommendation_server.py
View file

@ -14,13 +14,18 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import grpc
from concurrent import futures
import time
import traceback
import os import os
import random import random
import time
import traceback
from concurrent import futures
import googleclouddebugger import googleclouddebugger
import grpc
from opencensus.trace.exporters import print_exporter
from opencensus.trace.exporters import stackdriver_exporter
from opencensus.trace.ext.grpc import server_interceptor
from opencensus.trace.samplers import always_on
import demo_pb2 import demo_pb2
import demo_pb2_grpc import demo_pb2_grpc
@ -30,11 +35,6 @@ from grpc_health.v1 import health_pb2_grpc
from logger import getJSONLogger from logger import getJSONLogger
logger = getJSONLogger('recommendationservice-server') logger = getJSONLogger('recommendationservice-server')
# TODO(morganmclean,ahmetb) tracing currently disabled due to memory leak (see TODO below)
# from opencensus.trace.ext.grpc import server_interceptor
# from opencensus.trace.samplers import always_on
# from opencensus.trace.exporters import stackdriver_exporter
# from opencensus.trace.exporters import print_exporter
class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer): class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer):
def ListRecommendations(self, request, context): def ListRecommendations(self, request, context):
@ -63,15 +63,12 @@ class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer):
if __name__ == "__main__": if __name__ == "__main__":
logger.info("initializing recommendationservice") logger.info("initializing recommendationservice")
# TODO(morganmclean,ahmetb) enabling the tracing interceptor/sampler below try:
# causes an unbounded memory leak eventually OOMing the container. sampler = always_on.AlwaysOnSampler()
# ---- exporter = stackdriver_exporter.StackdriverExporter()
# try: tracer_interceptor = server_interceptor.OpenCensusServerInterceptor(sampler, exporter)
# sampler = always_on.AlwaysOnSampler() except:
# exporter = stackdriver_exporter.StackdriverExporter() tracer_interceptor = server_interceptor.OpenCensusServerInterceptor()
# tracer_interceptor = server_interceptor.OpenCensusServerInterceptor(sampler, exporter)
# except:
# tracer_interceptor = server_interceptor.OpenCensusServerInterceptor()
try: try:
googleclouddebugger.enable( googleclouddebugger.enable(

9
src/recommendationservice/requirements.txt
View file

@ -3,12 +3,11 @@ certifi==2018.4.16
chardet==3.0.4 chardet==3.0.4
enum34==1.1.6 enum34==1.1.6
futures==3.2.0 futures==3.2.0
google-api-core==1.2.1 google-api-core==1.6.0
google-api-python-client==1.7.4 google-api-python-client==1.7.4
google-auth==1.5.0 google-auth==1.6.1
google-auth-httplib2==0.0.3 google-auth-httplib2==0.0.3
google-cloud-core==0.28.1 google-cloud-core==0.29.0
google-cloud-trace==0.19.0
google-python-cloud-debugger==2.8 google-python-cloud-debugger==2.8
googleapis-common-protos==1.5.3 googleapis-common-protos==1.5.3
grpcio==1.13.0 grpcio==1.13.0
@ -16,7 +15,7 @@ grpcio-health-checking==1.13.0
grpcio-tools==1.0.0 grpcio-tools==1.0.0
httplib2==0.11.3 httplib2==0.11.3
idna==2.7 idna==2.7
opencensus==0.1.5 opencensus[stackdriver]==0.1.10
protobuf==3.5.2.post1 protobuf==3.5.2.post1
pyasn1==0.4.3 pyasn1==0.4.3
pyasn1-modules==0.2.2 pyasn1-modules==0.2.2