Use prometheus as a metric backend

This entails writing a metric aggregation program since each worker has its
own memory, and thus own metrics because of python gunicorn. The python
client is a simple wrapper that makes web requests to it.

data/queue.py

@@ -2,10 +2,14 @@ from datetime import datetime, timedelta
 
 from data.database import QueueItem, db, db_for_update, db_random_func
 from util.morecollections import AttrDict
+from util.prometheus import Gauge
 
 
 MINIMUM_EXTENSION = timedelta(seconds=20)
 
+build_capacity_shortage = Gauge('build_capacity_shortage', 'Build capacity shortage.')
+percent_building = Gauge('build_percent_building', 'Percent building.')
+
 class NoopWith:
   def __enter__(self):
     pass
@@ -20,9 +24,11 @@ class MetricQueueReporter(object):
   def __call__(self, currently_processing, running_count, total_count):
     need_capacity_count = total_count - running_count
     self._metric_queue.put('BuildCapacityShortage', need_capacity_count, unit='Count')
+    build_capacity_shortage.Set(need_capacity_count)
 
     building_percent = 100 if currently_processing else 0
     self._metric_queue.put('PercentBuilding', building_percent, unit='Percent')
+    percent_building.Set(building_percent)
 
 class WorkQueue(object):
   def __init__(self, queue_name, transaction_factory,