Add Stackdriver Profiler Python agent (#176)

* Add Stackdriver Profiler Python agent to EmailService and
RecommendationService

* Update recommendation_server.py

* Moved Profiler init to a function

* Moved Profiler init to a function

* Delete key.json

* Delete key.json

* Delete key.json

kubernetes-manifests/emailservice.yaml

@@ -39,6 +39,9 @@ spec:
           periodSeconds: 5
           exec:
             command: ["/bin/grpc_health_probe", "-addr=:8080"]
+        env:
+        - name: ENABLE_PROFILER
+          value: "0"
         resources:
           requests:
             cpu: 100m

kubernetes-manifests/recommendationservice.yaml

@@ -42,6 +42,8 @@ spec:
         env:
         - name: PRODUCT_CATALOG_SERVICE_ADDR
           value: "productcatalogservice:3550"
+        - name: ENABLE_PROFILER
+          value: "0"
         resources:
           requests:
             cpu: 100m

src/emailservice/Dockerfile

@@ -2,12 +2,16 @@ FROM python:3-slim as base
 
 FROM base as builder
 
+RUN apt-get -qq update \
+    && apt-get install -y --no-install-recommends \
+        g++ \
+    && rm -rf /var/lib/apt/lists/*
+
 # get packages
 COPY requirements.txt .
 RUN pip install -r requirements.txt
 
 FROM base as final
-
 # Enable unbuffered logging
 ENV PYTHONUNBUFFERED=1
 

src/emailservice/email_server.py

@@ -33,6 +33,7 @@ from opencensus.trace.ext.grpc import server_interceptor
 from opencensus.trace.samplers import always_on
 
 # import googleclouddebugger
+import googlecloudprofiler
 
 try:
     sampler = always_on.AlwaysOnSampler()
@@ -143,7 +144,41 @@ def start(dummy_mode):
   except KeyboardInterrupt:
     server.stop(0)
 
+def initStackdriverProfiling():
+  project_id = None
+  try:
+    project_id = os.environ["GCP_PROJECT_ID"]
+  except KeyError:
+    # Environment variable not set
+    pass
+
+  for retry in range(1,4):
+    try:
+      if project_id:
+        googlecloudprofiler.start(service='email_server', service_version='1.0.0', verbose=0, project_id=project_id)
+      else:
+        googlecloudprofiler.start(service='email_server', service_version='1.0.0', verbose=0)
+      logger.info("Successfully started Stackdriver Profiler.")
+      return
+    except (BaseException) as exc:
+      logger.info("Unable to start Stackdriver Profiler Python agent. " + str(exc))
+      if (retry < 4):
+        logger.info("Sleeping %d to retry initializing Stackdriver Profiler"%(retry*10))
+        time.sleep (1)
+      else:
+        logger.warning("Could not initialize Stackdriver Profiler after retrying, giving up")
+  return
 
 if __name__ == '__main__':
   logger.info('starting the email service in dummy mode.')
+  try:
+    enable_profiler = os.environ["ENABLE_PROFILER"]
+    if enable_profiler != "1":
+      raise KeyError()
+    else:
+      initStackdriverProfiling()
+  except KeyError:
+      logger.info("Skipping Stackdriver Profiler Python agent initialization. Set environment variable ENABLE_PROFILER=1 to enable.")
+
+
   start(dummy_mode = True)

src/emailservice/requirements.in

@@ -4,3 +4,4 @@ grpcio==1.16.1
 jinja2==2.10
 opencensus[stackdriver]==0.1.10
 python-json-logger==0.1.9
+google-cloud-profiler==1.0.8

src/emailservice/requirements.txt

@@ -8,22 +8,27 @@ cachetools==3.0.0         # via google-auth
 certifi==2018.11.29       # via requests
 chardet==3.0.4            # via requests
 google-api-core[grpc]==1.6.0
-google-auth==1.6.2        # via google-api-core
+google-api-python-client==1.7.8  # via google-cloud-profiler
+google-auth-httplib2==0.0.3  # via google-api-python-client, google-cloud-profiler
+google-auth==1.6.2        # via google-api-core, google-api-python-client, google-auth-httplib2, google-cloud-profiler
 google-cloud-core==0.29.1  # via google-cloud-trace
+google-cloud-profiler==1.0.8
 google-cloud-trace==0.20.2  # via opencensus
 googleapis-common-protos==1.5.5  # via google-api-core
 grpcio-health-checking==1.12.1
 grpcio==1.16.1
+httplib2==0.12.1          # via google-api-python-client, google-auth-httplib2
 idna==2.8                 # via requests
 jinja2==2.10
 markupsafe==1.1.0         # via jinja2
 opencensus[stackdriver]==0.1.10
-protobuf==3.6.1           # via google-api-core, googleapis-common-protos, grpcio-health-checking
+protobuf==3.6.1           # via google-api-core, google-cloud-profiler, googleapis-common-protos, grpcio-health-checking
 pyasn1-modules==0.2.3     # via google-auth
 pyasn1==0.4.5             # via pyasn1-modules, rsa
 python-json-logger==0.1.9
 pytz==2018.9              # via google-api-core
-requests==2.21.0          # via google-api-core
+requests==2.21.0          # via google-api-core, google-cloud-profiler
 rsa==4.0                  # via google-auth
-six==1.12.0               # via google-api-core, google-auth, grpcio, protobuf
+six==1.12.0               # via google-api-core, google-api-python-client, google-auth, grpcio, protobuf
+uritemplate==3.0.0        # via google-api-python-client
 urllib3==1.24.1           # via requests

src/recommendationservice/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:2.7-slim
 RUN apt-get update -qqy && \
-	apt-get -qqy install wget && \
+	apt-get -qqy install wget g++ && \
 	rm -rf /var/lib/apt/lists/*
 # show python logs as they occur
 ENV PYTHONUNBUFFERED=0

src/recommendationservice/recommendation_server.py

@@ -21,6 +21,7 @@ import traceback
 from concurrent import futures
 
 import googleclouddebugger
+import googlecloudprofiler
 import grpc
 from opencensus.trace.exporters import print_exporter
 from opencensus.trace.exporters import stackdriver_exporter
@@ -35,6 +36,30 @@ from grpc_health.v1 import health_pb2_grpc
 from logger import getJSONLogger
 logger = getJSONLogger('recommendationservice-server')
 
+def initStackdriverProfiling():
+  project_id = None
+  try:
+    project_id = os.environ["GCP_PROJECT_ID"]
+  except KeyError:
+    # Environment variable not set
+    pass
+
+  for retry in xrange(1,4):
+    try:
+      if project_id:
+        googlecloudprofiler.start(service='recommendation_server', service_version='1.0.0', verbose=0, project_id=project_id)
+      else:
+        googlecloudprofiler.start(service='recommendation_server', service_version='1.0.0', verbose=0)
+      logger.info("Successfully started Stackdriver Profiler.")
+      return
+    except (BaseException) as exc:
+      logger.info("Unable to start Stackdriver Profiler Python agent. " + str(exc))
+      if (retry < 4):
+        logger.info("Sleeping %d seconds to retry Stackdriver Profiler agent initialization"%(retry*10))
+        time.sleep (1)
+      else:
+        logger.warning("Could not initialize Stackdriver Profiler after retrying, giving up")
+  return
 
 class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer):
     def ListRecommendations(self, request, context):
@@ -63,6 +88,15 @@ class RecommendationService(demo_pb2_grpc.RecommendationServiceServicer):
 if __name__ == "__main__":
     logger.info("initializing recommendationservice")
 
+    try:
+      enable_profiler = os.environ["ENABLE_PROFILER"]
+      if enable_profiler != "1":
+        raise KeyError()
+      else:
+        initStackdriverProfiling()
+    except KeyError:
+      logger.info("Skipping Stackdriver Profiler Python agent initialization. Set environment variable ENABLE_PROFILER=1 to enable.")
+
     try:
         sampler = always_on.AlwaysOnSampler()
         exporter = stackdriver_exporter.StackdriverExporter()

src/recommendationservice/requirements.in

@@ -4,3 +4,4 @@ grpcio-health-checking==1.13.0
 grpcio==1.16.1
 opencensus[stackdriver]==0.1.10
 python-json-logger==0.1.9
+google-cloud-profiler==1.0.8

src/recommendationservice/requirements.txt

@@ -7,13 +7,12 @@
 cachetools==3.1.0         # via google-auth
 certifi==2018.11.29       # via requests
 chardet==3.0.4            # via requests
-enum34==1.1.6             # via grpcio
-futures==3.2.0            # via google-api-core, grpcio
 google-api-core[grpc]==1.6.0
-google-api-python-client==1.7.8  # via google-python-cloud-debugger
+google-api-python-client==1.7.8  # via google-cloud-profiler, google-python-cloud-debugger
-google-auth-httplib2==0.0.3  # via google-api-python-client, google-python-cloud-debugger
+google-auth-httplib2==0.0.3  # via google-api-python-client, google-cloud-profiler, google-python-cloud-debugger
-google-auth==1.6.2        # via google-api-core, google-api-python-client, google-auth-httplib2, google-python-cloud-debugger
+google-auth==1.6.2        # via google-api-core, google-api-python-client, google-auth-httplib2, google-cloud-profiler, google-python-cloud-debugger
 google-cloud-core==0.29.1  # via google-cloud-trace
+google-cloud-profiler==1.0.8
 google-cloud-trace==0.20.2  # via opencensus
 google-python-cloud-debugger==2.9
 googleapis-common-protos==1.5.6  # via google-api-core
@@ -22,13 +21,13 @@ grpcio==1.16.1
 httplib2==0.12.0          # via google-api-python-client, google-auth-httplib2
 idna==2.8                 # via requests
 opencensus[stackdriver]==0.1.10
-protobuf==3.6.1           # via google-api-core, googleapis-common-protos, grpcio-health-checking
+protobuf==3.6.1           # via google-api-core, google-cloud-profiler, googleapis-common-protos, grpcio-health-checking
 pyasn1-modules==0.2.4     # via google-auth
 pyasn1==0.4.5             # via pyasn1-modules, rsa
 python-json-logger==0.1.9
 pytz==2018.9              # via google-api-core
 pyyaml==3.13              # via google-python-cloud-debugger
-requests==2.21.0          # via google-api-core
+requests==2.21.0          # via google-api-core, google-cloud-profiler
 rsa==4.0                  # via google-auth
 six==1.12.0               # via google-api-core, google-api-python-client, google-auth, google-python-cloud-debugger, grpcio, protobuf
 uritemplate==3.0.0        # via google-api-python-client