122 lines
4.7 KiB
Python
122 lines
4.7 KiB
Python
import datetime
|
|
import logging
|
|
import time
|
|
|
|
from functools import wraps
|
|
from Queue import Queue, Full
|
|
|
|
from flask import g, request
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Buckets for the API response times.
|
|
API_RESPONSE_TIME_BUCKETS = [.01, .025, .05, .1, .25, .5, 1.0, 2.5, 5.0]
|
|
|
|
|
|
class MetricQueue(object):
|
|
""" Object to which various metrics are written, for distribution to metrics collection
|
|
system(s) such Prometheus.
|
|
"""
|
|
def __init__(self, prom):
|
|
# Define the various exported metrics.
|
|
self.resp_time = prom.create_histogram('response_time', 'HTTP response time in seconds',
|
|
labelnames=['endpoint'],
|
|
buckets=API_RESPONSE_TIME_BUCKETS)
|
|
self.resp_code = prom.create_counter('response_code', 'HTTP response code',
|
|
labelnames=['endpoint', 'code'])
|
|
self.non_200 = prom.create_counter('response_non200', 'Non-200 HTTP response codes',
|
|
labelnames=['endpoint'])
|
|
self.multipart_upload_start = prom.create_counter('multipart_upload_start',
|
|
'Multipart upload startse')
|
|
self.multipart_upload_end = prom.create_counter('self._metric_queue.multipart_upload_end',
|
|
'Multipart upload ends.', labelnames=['type'])
|
|
self.build_capacity_shortage = prom.create_gauge('build_capacity_shortage',
|
|
'Build capacity shortage.')
|
|
self.percent_building = prom.create_gauge('build_percent_building', 'Percent building.')
|
|
self.build_counter = prom.create_counter('builds', 'Number of builds', labelnames=['name'])
|
|
self.ephemeral_build_workers = prom.create_counter('ephemeral_build_workers',
|
|
'Number of started ephemeral build workers', labelnames=['name', 'build_uuid'])
|
|
self.ephemeral_build_worker_failure = prom.create_counter('ephemeral_build_worker_failure',
|
|
'Number of failed-to-start ephemeral build workers', labelnames=['build_uuid'])
|
|
|
|
self.work_queue_running = prom.create_gauge('work_queue_running', 'Running items in a queue',
|
|
labelnames=['queue_name'])
|
|
self.work_queue_available = prom.create_gauge('work_queue_available',
|
|
'Available items in a queue',
|
|
labelnames=['queue_name'])
|
|
|
|
# Deprecated: Define an in-memory queue for reporting metrics to CloudWatch or another
|
|
# provider.
|
|
self._queue = None
|
|
|
|
def enable_deprecated(self, maxsize=10000):
|
|
self._queue = Queue(maxsize)
|
|
|
|
def put_deprecated(self, name, value, **kwargs):
|
|
if self._queue is None:
|
|
logger.debug('No metric queue %s %s %s', name, value, kwargs)
|
|
return
|
|
|
|
try:
|
|
kwargs.setdefault('timestamp', datetime.datetime.now())
|
|
kwargs.setdefault('dimensions', {})
|
|
self._queue.put_nowait((name, value, kwargs))
|
|
except Full:
|
|
logger.error('Metric queue full')
|
|
|
|
def get_deprecated(self):
|
|
return self._queue.get()
|
|
|
|
def get_nowait_deprecated(self):
|
|
return self._queue.get_nowait()
|
|
|
|
|
|
def time_decorator(name, metric_queue):
|
|
""" Decorates an endpoint method to have its request time logged to the metrics queue. """
|
|
after = _time_after_request(name, metric_queue)
|
|
def decorator(func):
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
_time_before_request()
|
|
rv = func(*args, **kwargs)
|
|
after(rv)
|
|
return rv
|
|
return wrapper
|
|
return decorator
|
|
|
|
|
|
def time_blueprint(bp, metric_queue):
|
|
""" Decorates a blueprint to have its request time logged to the metrics queue. """
|
|
bp.before_request(_time_before_request)
|
|
bp.after_request(_time_after_request(bp.name, metric_queue))
|
|
|
|
|
|
def _time_before_request():
|
|
g._request_start_time = time.time()
|
|
|
|
|
|
def _time_after_request(name, metric_queue):
|
|
def f(r):
|
|
start = getattr(g, '_request_start_time', None)
|
|
if start is None:
|
|
return r
|
|
|
|
dur = time.time() - start
|
|
dims = {'endpoint': request.endpoint}
|
|
|
|
metric_queue.put_deprecated('ResponseTime', dur, dimensions=dims, unit='Seconds')
|
|
metric_queue.put_deprecated('ResponseCode', r.status_code, dimensions=dims)
|
|
|
|
metric_queue.resp_time.Observe(dur, labelvalues=[request.endpoint])
|
|
metric_queue.resp_code.Inc(labelvalues=[request.endpoint, r.status_code])
|
|
|
|
if r.status_code >= 500:
|
|
metric_queue.put_deprecated('5XXResponse', 1, dimensions={'name': name})
|
|
elif r.status_code < 200 or r.status_code >= 300:
|
|
metric_queue.put_deprecated('Non200Response', 1, dimensions={'name': name})
|
|
metric_queue.non_200.Inc(labelvalues=[request.endpoint])
|
|
|
|
return r
|
|
return f
|
|
|