From 8b706168467e9352e0c567633c3c264a340de79d Mon Sep 17 00:00:00 2001 From: tifayuki Date: Tue, 13 Feb 2018 13:30:56 -0800 Subject: [PATCH] Add notification metrics It adds notification related prometheus metrics, including: - total count for events/success/failure/error - total count for notification per each status code - gauge of the pending notification queue Signed-off-by: tifayuki --- metrics/prometheus.go | 3 +++ notifications/metrics.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/metrics/prometheus.go b/metrics/prometheus.go index b5a53214..91b32b23 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -10,4 +10,7 @@ const ( var ( // StorageNamespace is the prometheus namespace of blob/cache related operations StorageNamespace = metrics.NewNamespace(NamespacePrefix, "storage", nil) + + // NotificationsNamespace is the prometheus namespace of notification related metrics + NotificationsNamespace = metrics.NewNamespace(NamespacePrefix, "notifications", nil) ) diff --git a/notifications/metrics.go b/notifications/metrics.go index a20af168..69960e9c 100644 --- a/notifications/metrics.go +++ b/notifications/metrics.go @@ -5,6 +5,18 @@ import ( "fmt" "net/http" "sync" + + prometheus "github.com/docker/distribution/metrics" + "github.com/docker/go-metrics" +) + +var ( + // eventsCounter counts total events of incoming, success, failure, and errors + eventsCounter = prometheus.NotificationsNamespace.NewLabeledCounter("events", "The number of total events", "type") + // pendingGauge measures the pending queue size + pendingGauge = prometheus.NotificationsNamespace.NewGauge("pending", "The gauge of pending events in queue", metrics.Total) + // statusCounter counts the total notification call per each status code + statusCounter = prometheus.NotificationsNamespace.NewLabeledCounter("status", "The number of status code", "code") ) // EndpointMetrics track various actions taken by the endpoint, typically by @@ -61,6 +73,9 @@ func (emsl *endpointMetricsHTTPStatusListener) success(status int, events ...Eve defer emsl.safeMetrics.Unlock() emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) emsl.Successes += len(events) + + statusCounter.WithValues(fmt.Sprintf("%d %s", status, http.StatusText(status))).Inc(1) + eventsCounter.WithValues("Successes").Inc(1) } func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Event) { @@ -68,12 +83,17 @@ func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Eve defer emsl.safeMetrics.Unlock() emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) emsl.Failures += len(events) + + statusCounter.WithValues(fmt.Sprintf("%d %s", status, http.StatusText(status))).Inc(1) + eventsCounter.WithValues("Failures").Inc(1) } func (emsl *endpointMetricsHTTPStatusListener) err(err error, events ...Event) { emsl.safeMetrics.Lock() defer emsl.safeMetrics.Unlock() emsl.Errors += len(events) + + eventsCounter.WithValues("Errors").Inc(1) } // endpointMetricsEventQueueListener maintains the incoming events counter and @@ -87,12 +107,17 @@ func (eqc *endpointMetricsEventQueueListener) ingress(events ...Event) { defer eqc.Unlock() eqc.Events += len(events) eqc.Pending += len(events) + + eventsCounter.WithValues("Events").Inc() + pendingGauge.Inc(1) } func (eqc *endpointMetricsEventQueueListener) egress(events ...Event) { eqc.Lock() defer eqc.Unlock() eqc.Pending -= len(events) + + pendingGauge.Dec(1) } // endpoints is global registry of endpoints used to report metrics to expvar @@ -149,4 +174,7 @@ func init() { })) registry.(*expvar.Map).Set("notifications", ¬ifications) + + // register prometheus metrics + metrics.Register(prometheus.NotificationsNamespace) }