4bf4ce33c9
This change replaces the metricqueue library with a native Prometheus client implementation with the intention to aggregated results with the Prometheus PushGateway. This change also adds instrumentation for greenlet context switches.
82 lines
2.2 KiB
Python
82 lines
2.2 KiB
Python
import logging
|
|
import time
|
|
import threading
|
|
|
|
from flask import g, request
|
|
from prometheus_client import push_to_gateway, REGISTRY, Histogram
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
request_duration = Histogram('quay_request_duration_seconds',
|
|
'seconds taken to process a request',
|
|
labelnames=['method', 'endpoint', 'status'],
|
|
buckets=[.01, .025, .05, .1, .25, .5, 1.0, 2.5, 5.0])
|
|
|
|
|
|
PROMETHEUS_PUSH_INTERVAL_SECONDS = 30
|
|
ONE_DAY_IN_SECONDS = 60 * 60 * 24
|
|
|
|
|
|
class PrometheusPlugin(object):
|
|
""" Application plugin for reporting metrics to Prometheus. """
|
|
def __init__(self, app=None):
|
|
self.app = app
|
|
if app is not None:
|
|
self.state = self.init_app(app)
|
|
else:
|
|
self.state = None
|
|
|
|
def init_app(self, app):
|
|
pusher = ThreadPusher(app)
|
|
pusher.start()
|
|
|
|
# register extension with app
|
|
app.extensions = getattr(app, 'extensions', {})
|
|
app.extensions['prometheus'] = pusher
|
|
return pusher
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.state, name, None)
|
|
|
|
|
|
class ThreadPusher(threading.Thread):
|
|
def __init__(self, app):
|
|
super(ThreadPusher, self).__init__()
|
|
self.daemon = True
|
|
self._app = app
|
|
|
|
def run(self):
|
|
agg_url = self._app.config.get('PROMETHEUS_AGGREGATOR_URL')
|
|
while True:
|
|
if agg_url is None:
|
|
# Practically disable this worker, if there is no aggregator.
|
|
time.sleep(ONE_DAY_IN_SECONDS)
|
|
continue
|
|
|
|
time.sleep(PROMETHEUS_PUSH_INTERVAL_SECONDS)
|
|
push_to_gateway(agg_url, job=self._app.config.get('PROMETHEUS_NAMESPACE', 'quay'),
|
|
registry=REGISTRY)
|
|
|
|
|
|
def timed_blueprint(bp):
|
|
"""
|
|
Decorates a blueprint to have its request duration tracked by Prometheus.
|
|
"""
|
|
def _time_before_request():
|
|
g._request_start_time = time.time()
|
|
bp.before_request(_time_before_request)
|
|
|
|
|
|
def _time_after_request():
|
|
def f(r):
|
|
start = getattr(g, '_request_start_time', None)
|
|
if start is None:
|
|
return r
|
|
dur = time.time() - start
|
|
request_duration.labels(request.method, request.endpoint, r.status_code).observe(dur)
|
|
return r
|
|
return f
|
|
bp.after_request(_time_after_request())
|
|
return bp
|