Added runbook and updated makefile

Makefile

@@ -13,6 +13,16 @@ ZONE=australia-southeast1-a
 ISTIO_VERSION=1.5.0
 
 a: help
+all: all.cluster all.istio default.app
+
+all.cluster: cluster.create get.creds
+
+all.istio: ns.create.istio-system istio.init crd.wait ns.istio.enabled istio.template istio.deploy
+
+default.app: crd.wait skaffold.run.gcp.istio hipster.istio.rules
+
+crd.wait:
+	@kubectl -n istio-system wait --for=condition=complete job --all
 
 ## Use Istio Version 1.5.0
 istio150:
@@ -25,7 +35,7 @@ istio135:
 ## Create GKE Cluster with istio enabled
 cluster.create.istio:
  	@gcloud container clusters create ${CLUSTER_NAME} --enable-autoupgrade \
-	--enable-autoscaling --min-nodes=1 --max-nodes=10 --num-nodes=4 --zone=${ZONE} \
+	--enable-autoscaling --min-nodes=1 --max-nodes=10 --num-nodes=6 --zone=${ZONE} \
 	--addons=Istio --istio-config=auth=MTLS_PERMISSIVE \
 	--machine-type=n1-standard-2
 
@@ -43,7 +53,7 @@ cluster.resize:
 ## Create GKE Cluster
 cluster.create:
 	@gcloud container clusters create ${CLUSTER_NAME} --enable-autoupgrade \
-	--enable-autoscaling --min-nodes=1 --max-nodes=10 --num-nodes=4 --zone=${ZONE} \
+	--enable-autoscaling --min-nodes=1 --max-nodes=10 --num-nodes=6 --zone=${ZONE} \
 	--machine-type=n1-standard-2
 
 ## Get Cluster Creds
@@ -67,9 +77,6 @@ ns.istio.disabled:
 istio.init:
 	@helm template istio-${ISTIO_VERSION}/install/kubernetes/helm/istio-init --name istio-init --namespace istio-system | kubectl apply -f -
 
-istio.init.delete:
-	@helm template istio-${ISTIO_VERSION}/install/kubernetes/helm/istio-init --name istio-init --namespace istio-system | kubectl delete -f -
-
 
 ## Generate Istio Template
 istio.template:
@@ -77,9 +84,13 @@ istio.template:
 	--values istio-${ISTIO_VERSION}/install/kubernetes/helm/istio/values-istio-demo.yaml > istio-manifests/istio-demo.yaml
 
 ## Deploy Istio Config
-istio.deploy: istio.template
+istio.deploy:
 	@kubectl apply -f istio-manifests/istio-demo.yaml
 
+## Check if prometheus-stackdriver-sidecar has been deployed
+prom.sidecar.exist:
+	@kubectl -n istio-system get deployment prometheus -o=go-template='{{$output := "stackdriver-prometheus-sidecar does not exists."}}{{range .spec.template.spec.containers}}{{if eq .name "sidecar"}}{{$output = (print "stackdriver-prometheus-sidecar exists. Image: " .image)}}{{end}}{{end}}{{printf $output}}{{"\n"}}'
+
 ## Delete Istio Config
 istio.delete:
 	@kubectl delete -f istio-manifests/istio-demo.yaml
@@ -122,6 +133,15 @@ skaffold.build.gcp:
 cluster.delete:
 	@gcloud container clusters delete ${CLUSTER_NAME} --zone ${ZONE}
 
+## Application Istio Rules
+hipster.istio.rules:
+	@kubectl apply -f istio-manifests/frontend.yaml
+	@kubectl apply -f istio-manifests/frontend-gateway.yaml
+	@kubectl apply -f istio-manifests/whitelist-egress-googleapis.yaml
+
+istio.init.delete:
+	@helm template istio-${ISTIO_VERSION}/install/kubernetes/helm/istio-init --name istio-init --namespace istio-system | kubectl delete -f -
+
 help:
 	@echo ''
 	@echo 'Usage:'

README.md

@@ -289,3 +289,15 @@ resources.
 ---
 
 This is not an official Google project.
+
+============
+### Tony's notes
+
+## ref links
+
+[Cloud Monitoring metric export](https://cloud.google.com/solutions/stackdriver-monitoring-metric-export)
+
+
+[External metrics list](https://cloud.google.com/monitoring/api/metrics_other)
+
+[Metirc descriptors list](https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors/list)

istio-1.3.5/install/kubernetes/helm/istio/values-istio-demo.yaml

@@ -21,7 +21,7 @@ sidecarInjectorWebhook:
 
 pilot:
   autoscaleEnabled: false
-  traceSampling: 100.0
+  traceSampling: 10.0
   resources:
     requests:
       cpu: 10m

istio-manifests/istio-demo-prom-sidecar.yaml

@@ -0,0 +1,117 @@
+
+---
+# Source: istio/charts/prometheus/templates/deployment.yaml
+# TODO: the original template has service account, roles, etc
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: istio-system
+  labels:
+    app: prometheus
+    chart: prometheus
+    heritage: Tiller
+    release: istio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+        chart: prometheus
+        heritage: Tiller
+        release: istio
+      annotations:
+        sidecar.istio.io/inject: "false"
+    spec:
+      serviceAccountName: prometheus
+      containers:
+        - name: sidecar
+          image: gcr.io/stackdriver-prometheus/stackdriver-prometheus-sidecar:0.7.3
+          args:
+          - "--stackdriver.project-id=tonyh-gke-o11y-anz-openbanking"
+          - "--prometheus.wal-directory=/data/wal"
+          - "--prometheus.api-address=http://127.0.0.1:9090"
+          - "--stackdriver.kubernetes.location=australia-southeast1"
+          - "--stackdriver.kubernetes.cluster-name=o11y-ob"
+          ports:
+          - name: sidecar
+            containerPort: 9091
+          volumeMounts:
+          - name: data-volume
+            mountPath: /data
+        - name: prometheus
+          image: "docker.io/prom/prometheus:v2.12.0"
+          imagePullPolicy: IfNotPresent
+          args:
+            - '--storage.tsdb.retention=6h'
+            - '--config.file=/etc/prometheus/prometheus.yml'
+            - '--storage.tsdb.path=/data'
+          ports:
+            - containerPort: 9090
+              name: http
+          livenessProbe:
+            httpGet:
+              path: /-/healthy
+              port: 9090
+          readinessProbe:
+            httpGet:
+              path: /-/ready
+              port: 9090
+          resources:
+            requests:
+              cpu: 10m
+
+          volumeMounts:
+          - name: data-volume
+            mountPath: /data
+          - name: config-volume
+            mountPath: /etc/prometheus
+          - mountPath: /etc/istio-certs
+            name: istio-certs
+      volumes:
+      - name: data-volume
+        emptyDir: {}
+      - name: config-volume
+        configMap:
+          name: prometheus
+      - name: istio-certs
+        secret:
+          defaultMode: 420
+          secretName: istio.default
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - "amd64"
+                - "ppc64le"
+                - "s390x"
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 2
+            preference:
+              matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - "amd64"
+          - weight: 2
+            preference:
+              matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - "ppc64le"
+          - weight: 2
+            preference:
+              matchExpressions:
+              - key: beta.kubernetes.io/arch
+                operator: In
+                values:
+                - "s390x"

istio-manifests/istio-demo.yaml

@@ -15138,7 +15138,7 @@ data:
     accessLogFormat: ""
 
     # Set accessLogEncoding to JSON or TEXT to configure sidecar access log
-    accessLogEncoding: 'JSON'
+    accessLogEncoding: 'TEXT'
 
     enableEnvoyAccessLogService: false
     mixerCheckServer: istio-policy.istio-system.svc.cluster.local:9091
@@ -18239,7 +18239,7 @@ spec:
           - name: PILOT_PUSH_THROTTLE
             value: "100"
           - name: PILOT_TRACE_SAMPLING
-            value: "10"
+            value: "100"
           - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND
             value: "true"
           - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND
@@ -18376,27 +18376,12 @@ spec:
     spec:
       serviceAccountName: prometheus
       containers:
-        - name: sidecar
-          image: gcr.io/stackdriver-prometheus/stackdriver-prometheus-sidecar:0.7.3
-          args:
-          - "--stackdriver.project-id=tonyh-gke-o11y-anz-openbanking"
-          - "--prometheus.wal-directory=/data/wal"
-          - "--prometheus.api-address=http://127.0.0.1:9090"
-          - "--stackdriver.kubernetes.location=australia-southeast1"
-          - "--stackdriver.kubernetes.cluster-name=o11y-ob"
-          ports:
-          - name: sidecar
-            containerPort: 9091
-          volumeMounts:
-          - name: data-volume
-            mountPath: /data
         - name: prometheus
           image: "docker.io/prom/prometheus:v2.12.0"
           imagePullPolicy: IfNotPresent
           args:
             - '--storage.tsdb.retention=6h'
             - '--config.file=/etc/prometheus/prometheus.yml'
-            - '--storage.tsdb.path=/data'
           ports:
             - containerPort: 9090
               name: http
@@ -18413,15 +18398,11 @@ spec:
               cpu: 10m
             
           volumeMounts:
-          - name: data-volume
-            mountPath: /data
           - name: config-volume
             mountPath: /etc/prometheus
           - mountPath: /etc/istio-certs
             name: istio-certs
       volumes:
-      - name: data-volume
-        emptyDir: {}
       - name: config-volume
         configMap:
           name: prometheus

runbook.md

@@ -0,0 +1,49 @@
+# Runbook get demo up and running
+
+1. Update Variables in Makefile
+
+## GKE Cluster
+2. Create a cluster that isn't using Istio enabled.
+
+    `make cluster.create`
+
+> Takes a few minutes
+
+3. Get Cluster creds and set kubeconfig
+
+    `make get.creds`
+
+4.  Confirm connection to the cluster
+
+    `kubectl get nodes`
+
+### Istio Setup
+
+5. Create istio-system namespace
+
+   `make ns.create.istio-system`
+
+6. Initliase Istio
+
+    `make istio.init`
+
+7. Enable istio sidecar injection to default namesapce
+
+    `make ns.istio.enabled`
+
+8. Create Istio template (if no istio-demo.yaml)
+
+    `make istio.template`
+
+9. Deploy Istio
+
+    `make istio.deploy`
+> Wait for containers for istio to be deployed.
+
+    `kubectl get pods -n istio-system`
+
+## Application
+
+10. Deploy Istio configured app with tracing turned on
+
+     `make skaffold.run.gcp.istio`

skaffold.yaml

@@ -86,3 +86,5 @@ profiles:
     kubectl:
       manifests:
       - ./kubernetes-manifests-tracing/**.yaml
+      - ./istio-manifests/frontend**.yaml
+      - ./istio-manifests/whitelist-egress-googleapis.yaml