diff --git a/kubernetes-manifests/emailservice.yaml b/kubernetes-manifests/emailservice.yaml index c5dd7d8..179db0e 100644 --- a/kubernetes-manifests/emailservice.yaml +++ b/kubernetes-manifests/emailservice.yaml @@ -39,6 +39,9 @@ spec: periodSeconds: 5 exec: command: ["/bin/grpc_health_probe", "-addr=:8080"] + env: + - name: ENABLE_PROFILER + value: "0" resources: requests: cpu: 100m diff --git a/kubernetes-manifests/recommendationservice.yaml b/kubernetes-manifests/recommendationservice.yaml index 07d4c27..eeb7782 100644 --- a/kubernetes-manifests/recommendationservice.yaml +++ b/kubernetes-manifests/recommendationservice.yaml @@ -42,6 +42,8 @@ spec: env: - name: PRODUCT_CATALOG_SERVICE_ADDR value: "productcatalogservice:3550" + - name: ENABLE_PROFILER + value: "0" resources: requests: cpu: 100m diff --git a/src/emailservice/Dockerfile b/src/emailservice/Dockerfile index b4b2a94..3d649a5 100644 --- a/src/emailservice/Dockerfile +++ b/src/emailservice/Dockerfile @@ -2,12 +2,16 @@ FROM python:3-slim as base FROM base as builder +RUN apt-get -qq update \ + && apt-get install -y --no-install-recommends \ + g++ \ + && rm -rf /var/lib/apt/lists/* + # get packages COPY requirements.txt . RUN pip install -r requirements.txt FROM base as final - # Enable unbuffered logging ENV PYTHONUNBUFFERED=1 diff --git a/src/emailservice/email_server.py b/src/emailservice/email_server.py index f4a2162..5af5ff9 100644 --- a/src/emailservice/email_server.py +++ b/src/emailservice/email_server.py @@ -33,6 +33,7 @@ from opencensus.trace.ext.grpc import server_interceptor from opencensus.trace.samplers import always_on # import googleclouddebugger +import googlecloudprofiler try: sampler = always_on.AlwaysOnSampler() @@ -143,7 +144,41 @@ def start(dummy_mode): except KeyboardInterrupt: server.stop(0) +def initStackdriverProfiling(): + project_id = None + try: + project_id = os.environ["GCP_PROJECT_ID"] + except KeyError: + # Environment variable not set + pass + + for retry in range(1,4): + try: + if project_id: + googlecloudprofiler.start(service='email_server', service_version='1.0.0', verbose=0, project_id=project_id) + else: + googlecloudprofiler.start(service='email_server', service_version='1.0.0', verbose=0) + logger.info("Successfully started Stackdriver Profiler.") + return + except (BaseException) as exc: + logger.info("Unable to start Stackdriver Profiler Python agent. " + str(exc)) + if (retry < 4): + logger.info("Sleeping %d to retry initializing Stackdriver Profiler"%(retry*10)) + time.sleep (1) + else: + logger.warning("Could not initialize Stackdriver Profiler after retrying, giving up") + return if __name__ == '__main__': logger.info('starting the email service in dummy mode.') + try: + enable_profiler = os.environ["ENABLE_PROFILER"] + if enable_profiler != "1": + raise KeyError() + else: + initStackdriverProfiling() + except KeyError: + logger.info("Skipping Stackdriver Profiler Python agent initialization. Set environment variable ENABLE_PROFILER=1 to enable.") + + start(dummy_mode = True) diff --git a/src/emailservice/requirements.in b/src/emailservice/requirements.in index fa3f39c..bb60d5a 100644 --- a/src/emailservice/requirements.in +++ b/src/emailservice/requirements.in @@ -4,3 +4,4 @@ grpcio==1.16.1 jinja2==2.10 opencensus[stackdriver]==0.1.10 python-json-logger==0.1.9 +google-cloud-profiler==1.0.8 diff --git a/src/emailservice/requirements.txt b/src/emailservice/requirements.txt index 6a763b2..c0d5346 100644 --- a/src/emailservice/requirements.txt +++ b/src/emailservice/requirements.txt @@ -8,22 +8,27 @@ cachetools==3.0.0 # via google-auth certifi==2018.11.29 # via requests chardet==3.0.4 # via requests google-api-core[grpc]==1.6.0 -google-auth==1.6.2 # via google-api-core +google-api-python-client==1.7.8 # via google-cloud-profiler +google-auth-httplib2==0.0.3 # via google-api-python-client, google-cloud-profiler +google-auth==1.6.2 # via google-api-core, google-api-python-client, google-auth-httplib2, google-cloud-profiler google-cloud-core==0.29.1 # via google-cloud-trace +google-cloud-profiler==1.0.8 google-cloud-trace==0.20.2 # via opencensus googleapis-common-protos==1.5.5 # via google-api-core grpcio-health-checking==1.12.1 grpcio==1.16.1 +httplib2==0.12.1 # via google-api-python-client, google-auth-httplib2 idna==2.8 # via requests jinja2==2.10 markupsafe==1.1.0 # via jinja2 opencensus[stackdriver]==0.1.10 -protobuf==3.6.1 # via google-api-core, googleapis-common-protos, grpcio-health-checking +protobuf==3.6.1 # via google-api-core, google-cloud-profiler, googleapis-common-protos, grpcio-health-checking pyasn1-modules==0.2.3 # via google-auth pyasn1==0.4.5 # via pyasn1-modules, rsa python-json-logger==0.1.9 pytz==2018.9 # via google-api-core -requests==2.21.0 # via google-api-core +requests==2.21.0 # via google-api-core, google-cloud-profiler rsa==4.0 # via google-auth -six==1.12.0 # via google-api-core, google-auth, grpcio, protobuf +six==1.12.0 # via google-api-core, google-api-python-client, google-auth, grpcio, protobuf +uritemplate==3.0.0 # via google-api-python-client urllib3==1.24.1 # via requests diff --git a/src/recommendationservice/Dockerfile b/src/recommendationservice/Dockerfile index 3faff32..fc70bec 100644 --- a/src/recommendationservice/Dockerfile +++ b/src/recommendationservice/Dockerfile @@ -1,6 +1,6 @@ FROM python:2.7-slim RUN apt-get update -qqy && \ - apt-get -qqy install wget && \ + apt-get -qqy install wget g++ && \ rm -rf /var/lib/apt/lists/* # show python logs as they occur ENV PYTHONUNBUFFERED=0 diff --git a/src/recommendationservice/recommendation_server.py b/src/recommendationservice/recommendation_server.py index eb24f96..357f25f 100644 --- a/src/recommendationservice/recommendation_server.py +++ b/src/recommendationservice/recommendation_server.py @@ -21,6 +21,7 @@ import traceback from concurrent import futures import googleclouddebugger +import googlecloudprofiler import grpc from opencensus.trace.exporters import print_exporter from opencensus.trace.exporters import stackdriver_exporter @@ -35,6 +36,30 @@ from grpc_health.v1 import health_pb2_grpc from logger import getJSONLogger logger = getJSONLogger('recommendationservice-server') +def initStackdriverProfiling(): + project_id = None + try: + project_id = os.environ["GCP_PROJECT_ID"] + except KeyError: + # Environment variable not set + pass + + for retry in xrange(1,4): + try: + if project_id: + googlecloudprofiler.start(service='recommendation_server', service_version='1.0.0', verbose=0, project_id=project_id) + else: + googlecloudprofiler.start(service='recommendation_server', service_version='1.0.0', verbose=0) + logger.info("Successfully started Stackdriver Profiler.") + return + except (BaseException) as exc: + logger.info("Unable to start Stackdriver Profiler Python agent. " + str(exc)) + if (retry < 4): + logger.info("Sleeping %d seconds to retry Stackdriver Profiler agent initialization"%(retry*10)) + time.sleep (1) + else: + logger.warning("Could not initialize Stackdriver Profiler after retrying, giving up") + return class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer): def ListRecommendations(self, request, context): @@ -63,6 +88,15 @@ class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer): if __name__ == "__main__": logger.info("initializing recommendationservice") + try: + enable_profiler = os.environ["ENABLE_PROFILER"] + if enable_profiler != "1": + raise KeyError() + else: + initStackdriverProfiling() + except KeyError: + logger.info("Skipping Stackdriver Profiler Python agent initialization. Set environment variable ENABLE_PROFILER=1 to enable.") + try: sampler = always_on.AlwaysOnSampler() exporter = stackdriver_exporter.StackdriverExporter() diff --git a/src/recommendationservice/requirements.in b/src/recommendationservice/requirements.in index 10d437d..bfc28a5 100644 --- a/src/recommendationservice/requirements.in +++ b/src/recommendationservice/requirements.in @@ -4,3 +4,4 @@ grpcio-health-checking==1.13.0 grpcio==1.16.1 opencensus[stackdriver]==0.1.10 python-json-logger==0.1.9 +google-cloud-profiler==1.0.8 diff --git a/src/recommendationservice/requirements.txt b/src/recommendationservice/requirements.txt index 50114b2..b504fff 100644 --- a/src/recommendationservice/requirements.txt +++ b/src/recommendationservice/requirements.txt @@ -7,13 +7,12 @@ cachetools==3.1.0 # via google-auth certifi==2018.11.29 # via requests chardet==3.0.4 # via requests -enum34==1.1.6 # via grpcio -futures==3.2.0 # via google-api-core, grpcio google-api-core[grpc]==1.6.0 -google-api-python-client==1.7.8 # via google-python-cloud-debugger -google-auth-httplib2==0.0.3 # via google-api-python-client, google-python-cloud-debugger -google-auth==1.6.2 # via google-api-core, google-api-python-client, google-auth-httplib2, google-python-cloud-debugger +google-api-python-client==1.7.8 # via google-cloud-profiler, google-python-cloud-debugger +google-auth-httplib2==0.0.3 # via google-api-python-client, google-cloud-profiler, google-python-cloud-debugger +google-auth==1.6.2 # via google-api-core, google-api-python-client, google-auth-httplib2, google-cloud-profiler, google-python-cloud-debugger google-cloud-core==0.29.1 # via google-cloud-trace +google-cloud-profiler==1.0.8 google-cloud-trace==0.20.2 # via opencensus google-python-cloud-debugger==2.9 googleapis-common-protos==1.5.6 # via google-api-core @@ -22,13 +21,13 @@ grpcio==1.16.1 httplib2==0.12.0 # via google-api-python-client, google-auth-httplib2 idna==2.8 # via requests opencensus[stackdriver]==0.1.10 -protobuf==3.6.1 # via google-api-core, googleapis-common-protos, grpcio-health-checking +protobuf==3.6.1 # via google-api-core, google-cloud-profiler, googleapis-common-protos, grpcio-health-checking pyasn1-modules==0.2.4 # via google-auth pyasn1==0.4.5 # via pyasn1-modules, rsa python-json-logger==0.1.9 pytz==2018.9 # via google-api-core pyyaml==3.13 # via google-python-cloud-debugger -requests==2.21.0 # via google-api-core +requests==2.21.0 # via google-api-core, google-cloud-profiler rsa==4.0 # via google-auth six==1.12.0 # via google-api-core, google-api-python-client, google-auth, google-python-cloud-debugger, grpcio, protobuf uritemplate==3.0.0 # via google-api-python-client