Add a queue capacity reporter plugin to the queue. Move the queue definitions to app. Add a cloudwatch reporter to the dockerfile build queue.

util/queuemetrics.py

@@ -0,0 +1,51 @@
+import logging
+import boto
+
+
+logger = logging.getLogger(__name__)
+
+
+class NullReporter(object):
+  def report(self, running_count, total_count):
+    pass
+
+
+class CloudWatchReporter(object):
+  def __init__(self, aws_access_key, aws_secret_key, namespace, name):
+    self._connection = boto.connect_cloudwatch(aws_access_key, aws_secret_key)
+    self._namespace = namespace
+    self._name = name
+
+  def report(self, running_count, total_count):
+    need_capacity_count = total_count - running_count
+    self._connection.put_metric_data(self._namespace, self._name, need_capacity_count,
+                                     unit='Count')
+
+
+class QueueMetrics(object):
+  def __init__(self, app=None):
+    self.app = app
+    if app is not None:
+      self.state = self.init_app(app)
+    else:
+      self.state = None
+
+  def init_app(self, app):
+    analytics_type = app.config.get('QUEUE_METRICS_TYPE', 'Null')
+
+    if analytics_type == 'CloudWatch':
+      access_key = app.config.get('QUEUE_METRICS_AWS_ACCESS_KEY', '')
+      secret_key = app.config.get('QUEUE_METRICS_AWS_SECRET_KEY', '')
+      namespace = app.config.get('QUEUE_METRICS_NAMESPACE', '')
+      name = app.config.get('QUEUE_METRICS_NAME', '')
+      reporter = CloudWatchReporter(access_key, secret_key, namespace, name)
+    else:
+      reporter = NullReporter()
+
+    # register extension with app
+    app.extensions = getattr(app, 'extensions', {})
+    app.extensions['queuemetrics'] = reporter
+    return reporter
+
+  def __getattr__(self, name):
+    return getattr(self.state, name, None)