Add duration metric collector decorator (#1885)

Track time-to-start for builders
Track time-to-build for builders
Track ec2 builder fallbacks
Track build time

test/test_buildman.py

@@ -13,7 +13,8 @@ from buildman.manager.ephemeral import (EphemeralBuilderManager, EtcdAction,
                                         ETCD_MAX_WATCH_TIMEOUT)
 from buildman.component.buildcomponent import BuildComponent
 from buildman.server import BuildJobResult
-
+from util.metrics.metricqueue import duration_collector_async
+from app import metric_queue
 
 BUILD_UUID = 'deadbeef-dead-beef-dead-deadbeefdead'
 REALM_ID = '1234-realm'
@@ -33,6 +34,7 @@ class TestExecutor(BuilderExecutor):
   job_stopped = None
 
   @coroutine
+  @duration_collector_async(metric_queue.builder_time_to_start, labelvalues=["testlabel"])
   def start_builder(self, realm, token, build_uuid):
     self.job_started = str(uuid.uuid4())
     raise Return(self.job_started)
@@ -45,6 +47,7 @@ class TestExecutor(BuilderExecutor):
 
 class BadExecutor(BuilderExecutor):
   @coroutine
+  @duration_collector_async(metric_queue.builder_time_to_start, labelvalues=["testlabel"])
   def start_builder(self, realm, token, build_uuid):
     raise ExecutorException('raised on purpose!')
 
@@ -210,6 +213,7 @@ class TestEphemeralLifecycle(EphemeralBuilderTestCase):
     # Take the job ourselves
     yield From(self.manager.build_component_ready(test_component))
 
+    self.etcd_client_mock.read.assert_called_with(os.path.join('realm/', REALM_ID))
     self.etcd_client_mock.delete.assert_called_once_with(os.path.join('realm/', REALM_ID))
     self.etcd_client_mock.delete.reset_mock()
 
@@ -743,4 +747,3 @@ class TestEphemeral(EphemeralBuilderTestCase):
 
 if __name__ == '__main__':
   unittest.main()
-

test/test_metricqueue.py

@@ -0,0 +1,50 @@
+import time
+import unittest
+
+from mock import Mock
+from trollius import coroutine, Return, get_event_loop
+
+from util.metrics.metricqueue import duration_collector_async
+
+
+mock_histogram = Mock()
+
+class NonReturn(Exception):
+  pass 
+
+
+@coroutine
+@duration_collector_async(mock_histogram, labelvalues=["testlabel"])
+def duration_decorated():
+    time.sleep(1)
+    raise Return("fin")
+
+
+@coroutine
+@duration_collector_async(mock_histogram, labelvalues=["testlabel"])
+def duration_decorated_error():
+    raise NonReturn("not a Return error")
+
+
+class DurationDecoratorTestCase(unittest.TestCase):
+  def __init__(self, *args, **kwargs):
+    self.etcd_client_mock = None
+    self.loop = get_event_loop()
+    super(DurationDecoratorTestCase, self).__init__(*args, **kwargs)
+
+  def test_duration_decorator(self):
+    self.loop.run_until_complete(duration_decorated())
+    assert mock_histogram.Observe.called
+    assert 1 - mock_histogram.Observe.call_args[0][0] < 1  # duration should be close to 1s
+    assert mock_histogram.Observe.call_args[1]["labelvalues"] == ["testlabel"]
+    
+  def test_duration_decorator_error(self):
+    mock_histogram.reset_mock()
+    
+    with self.assertRaises(NonReturn):
+      self.loop.run_until_complete(duration_decorated_error())
+    assert not mock_histogram.Observe.called
+
+
+if __name__ == '__main__':
+  unittest.main()