initial import for Open Source 🎉

util/saas/cloudwatch.py

@@ -0,0 +1,96 @@
+import logging
+import boto
+import time
+import random
+
+from Queue import Empty
+from threading import Thread
+
+
+logger = logging.getLogger(__name__)
+
+MAX_BATCH_METRICS = 20
+
+# Sleep for this much time between failed send requests.
+# This prevents hammering cloudwatch when it's not available.
+FAILED_SEND_SLEEP_SECS = 15
+
+def start_cloudwatch_sender(metrics, app):
+  """
+  Starts sending from metrics to a new CloudWatchSender.
+  """
+  access_key = app.config.get('CLOUDWATCH_AWS_ACCESS_KEY')
+  secret_key = app.config.get('CLOUDWATCH_AWS_SECRET_KEY')
+  namespace = app.config.get('CLOUDWATCH_NAMESPACE')
+
+  if not namespace:
+    logger.debug('CloudWatch not configured')
+    return
+
+  sender = CloudWatchSender(metrics, access_key, secret_key, namespace)
+  sender.start()
+
+class CloudWatchSender(Thread):
+  """
+  CloudWatchSender loops indefinitely and pulls metrics off of a queue then sends it to CloudWatch.
+  """
+  def __init__(self, metrics, aws_access_key, aws_secret_key, namespace):
+    Thread.__init__(self)
+    self.daemon = True
+
+    self._aws_access_key = aws_access_key
+    self._aws_secret_key = aws_secret_key
+    self._metrics = metrics
+    self._namespace = namespace
+
+  def run(self):
+    try:
+      logger.debug('Starting CloudWatch sender process.')
+      connection = boto.connect_cloudwatch(self._aws_access_key, self._aws_secret_key)
+    except:
+      logger.exception('Failed to connect to CloudWatch.')
+    self._metrics.enable_deprecated()
+
+    while True:
+      metrics = {
+        'name': [],
+        'value': [],
+        'unit': [],
+        'timestamp': [],
+        'dimensions': [],
+      }
+
+      metric = self._metrics.get_deprecated()
+      append_metric(metrics, metric)
+
+      while len(metrics['name']) < MAX_BATCH_METRICS:
+        try:
+          metric = self._metrics.get_nowait_deprecated()
+          append_metric(metrics, metric)
+        except Empty:
+          break
+
+      try:
+        connection.put_metric_data(self._namespace, **metrics)
+        logger.debug('Sent %d CloudWatch metrics', len(metrics['name']))
+      except:
+        for i in range(len(metrics['name'])):
+          self._metrics.put_deprecated(metrics['name'][i], metrics['value'][i],
+            unit=metrics['unit'][i],
+            dimensions=metrics['dimensions'][i],
+            timestamp=metrics['timestamp'][i],
+          )
+
+        logger.exception('Failed to write to CloudWatch: %s', metrics)
+        logger.debug('Attempted to requeue %d metrics.', len(metrics['name']))
+        # random int between 1/2 and 1 1/2 of FAILED_SEND_SLEEP duration
+        sleep_secs = random.randint(FAILED_SEND_SLEEP_SECS/2, 3*FAILED_SEND_SLEEP_SECS/2)
+        time.sleep(sleep_secs)
+
+def append_metric(metrics, m):
+  name, value, kwargs = m
+  metrics['name'].append(name)
+  metrics['value'].append(value)
+  metrics['unit'].append(kwargs.get('unit'))
+  metrics['dimensions'].append(kwargs.get('dimensions'))
+  metrics['timestamp'].append(kwargs.get('timestamp'))