Add duration metric collector decorator (#1885)

Track time-to-start for builders
Track time-to-build for builders
Track ec2 builder fallbacks
Track build time

util/metrics/metricqueue.py

@@ -6,6 +6,7 @@ from functools import wraps
 from Queue import Queue, Full
 
 from flask import g, request
+from trollius import Return
 
 
 logger = logging.getLogger(__name__)
@@ -13,6 +14,9 @@ logger = logging.getLogger(__name__)
 # Buckets for the API response times.
 API_RESPONSE_TIME_BUCKETS = [.01, .025, .05, .1, .25, .5, 1.0, 2.5, 5.0]
 
+# Buckets for the builder start times.
+BUILDER_START_TIME_BUCKETS = [.5, 1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 180.0, 240.0, 300.0, 600.0]
+
 
 class MetricQueue(object):
   """ Object to which various metrics are written, for distribution to metrics collection
@@ -33,6 +37,16 @@ class MetricQueue(object):
                                                     'Multipart upload ends.', labelnames=['type'])
     self.build_capacity_shortage = prom.create_gauge('build_capacity_shortage',
                                                      'Build capacity shortage.')
+    self.builder_time_to_start = prom.create_histogram('builder_tts',
+                                                       'Time from triggering to starting a builder.',
+                                                       labelnames=['builder_type'],
+                                                       buckets=BUILDER_START_TIME_BUCKETS)
+    self.builder_time_to_build = prom.create_histogram('builder_ttb', 
+                                                       'Time from triggering to actually starting a build',
+                                                       labelnames=['builder_type'],
+                                                       buckets=BUILDER_START_TIME_BUCKETS)
+    self.build_time = prom.create_histogram('build_time', 'Time spent buildig', labelnames=['builder_type'])
+    self.builder_fallback = prom.create_counter('builder_fallback', 'Builder fell back to ec2 executor')
     self.percent_building = prom.create_gauge('build_percent_building', 'Percent building.')
     self.build_counter = prom.create_counter('builds', 'Number of builds', labelnames=['name'])
     self.ephemeral_build_workers = prom.create_counter('ephemeral_build_workers',
@@ -88,6 +102,22 @@ class MetricQueue(object):
     return self._queue.get_nowait()
 
 
+def duration_collector_async(metric, labelvalues):
+  """ Decorates a method to have its duration time logged to the metric. """
+  def decorator(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+      trigger_time = time.time()
+      try:
+        rv = func(*args, **kwargs)
+      except Return as e:
+        metric.Observe(time.time() - trigger_time, labelvalues=labelvalues)
+        raise e
+      return rv
+    return wrapper
+  return decorator
+
+
 def time_decorator(name, metric_queue):
   """ Decorates an endpoint method to have its request time logged to the metrics queue. """
   after = _time_after_request(name, metric_queue)
@@ -135,4 +165,3 @@ def _time_after_request(name, metric_queue):
 
     return r
   return f
-