conf: replace prometheus aggregator w/ pushgateway

This change replaces the homegrown Prometheus aggregation process that runs inside the container with the upstream Prometheus PushGateway.
2019-11-13 15:11:59 -05:00 · 2019-11-13 15:11:59 -05:00 · fa1deff1ad
parent 4bf4ce33c9
commit fa1deff1ad
17 changed files with 70 additions and 55 deletions
--- a/19
+++ b/19
@ -1,7 +1,9 @@
 FROM centos:7
 LABEL maintainer "thomasmckay@redhat.com"

-ENV PYTHON_VERSION=2.7 \
+ENV OS=linux \
+    ARCH=amd64 \
+    PYTHON_VERSION=2.7 \
    PATH=$HOME/.local/bin/:$PATH \
    PYTHONUNBUFFERED=1 \
    PYTHONIOENCODING=UTF-8 \
@ -76,13 +78,18 @@ RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \

 # TODO: Build jwtproxy in dist-git
 #       https://jira.coreos.com/browse/QUAY-1315
-RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \
+ENV JWTPROXY_VERSION=0.0.3
+RUN curl -fsSL -o /usr/local/bin/jwtproxy "https://github.com/coreos/jwtproxy/releases/download/v${JWTPROXY_VERSION}/jwtproxy-${OS}-${ARCH}" && \
    chmod +x /usr/local/bin/jwtproxy

-# TODO: Build prometheus-aggregator in dist-git
+# TODO: Build pushgateway in dist-git
 #       https://jira.coreos.com/browse/QUAY-1324
-RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\
-    chmod +x /usr/local/bin/prometheus-aggregator
+ENV PUSHGATEWAY_VERSION=1.0.0
+RUN curl -fsSL "https://github.com/prometheus/pushgateway/releases/download/v${PUSHGATEWAY_VERSION}/pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}.tar.gz" | \
+    tar xz "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" && \
+    mv "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" /usr/local/bin/pushgateway && \
+    rm -rf "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}" && \
+    chmod +x /usr/local/bin/pushgateway

 # Update local copy of AWS IP Ranges.
 RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
@ -105,7 +112,7 @@ RUN UNINSTALL_PKGS="\
    yum clean all && \
    rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache

-EXPOSE 8080 8443 7443
+EXPOSE 8080 8443 7443 9091

 RUN chgrp -R 0 $QUAYDIR && \
    chmod -R g=u $QUAYDIR
--- a/Dockerfile.rhel7
+++ b/Dockerfile.rhel7
@ -1,7 +1,9 @@
 FROM registry.redhat.io/rhel7:7.7
 LABEL maintainer "thomasmckay@redhat.com"

-ENV PYTHON_VERSION=2.7 \
+ENV OS=linux \
+    ARCH=amd64 \
+    PYTHON_VERSION=2.7 \
    PATH=$HOME/.local/bin/:$PATH \
    PYTHONUNBUFFERED=1 \
    PYTHONIOENCODING=UTF-8 \
@ -81,13 +83,18 @@ RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \

 # TODO: Build jwtproxy in dist-git
 #       https://jira.coreos.com/browse/QUAY-1315
-RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \
+ENV JWTPROXY_VERSION=0.0.3
+RUN curl -fsSL -o /usr/local/bin/jwtproxy "https://github.com/coreos/jwtproxy/releases/download/v${JWTPROXY_VERSION}/jwtproxy-${OS}-${ARCH}" && \
    chmod +x /usr/local/bin/jwtproxy

-# TODO: Build prometheus-aggregator in dist-git
+# TODO: Build pushgateway in dist-git
 #       https://jira.coreos.com/browse/QUAY-1324
-RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\
-    chmod +x /usr/local/bin/prometheus-aggregator
+ENV PUSHGATEWAY_VERSION=1.0.0
+RUN curl -fsSL "https://github.com/prometheus/pushgateway/releases/download/v${PUSHGATEWAY_VERSION}/pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}.tar.gz" | \
+    tar xz "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" && \
+    mv "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" /usr/local/bin/pushgateway && \
+    rm -rf "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}" && \
+    chmod +x /usr/local/bin/pushgateway

 # Update local copy of AWS IP Ranges.
 RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
@ -110,7 +117,7 @@ RUN UNINSTALL_PKGS="\
    yum clean all && \
    rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache

-EXPOSE 8080 8443 7443
+EXPOSE 8080 8443 7443 9091

 RUN chgrp -R 0 $QUAYDIR && \
    chmod -R g=u $QUAYDIR
--- a/conf/init/service/interactive/prometheus-aggregator/log/run
+++ b/conf/init/service/interactive/prometheus-aggregator/log/run
@ -1,4 +0,0 @@
-#!/bin/sh
-
-# Start the logger
-exec logger -i -t prometheus-aggregator
--- a/conf/init/service/interactive/prometheus-aggregator/run
+++ b/conf/init/service/interactive/prometheus-aggregator/run
@ -1,7 +0,0 @@
-#! /bin/bash
-
-echo 'Starting prometheus aggregator'
-
-/usr/local/bin/prometheus-aggregator
-
-echo 'Prometheus aggregator exited'
--- a/conf/init/service/interactive/pushgateway/log/run
+++ b/conf/init/service/interactive/pushgateway/log/run
@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Start the logger
+exec logger -i -t pushgateway
--- a/conf/init/service/interactive/pushgateway/run
+++ b/conf/init/service/interactive/pushgateway/run
@ -0,0 +1,7 @@
+#! /bin/bash
+
+echo 'Starting prometheus pushgateway'
+
+/usr/local/bin/pushgateway
+
+echo 'Prometheus pushgateway exited'
--- a/conf/init/supervisord_conf_create.py
+++ b/conf/init/supervisord_conf_create.py
@ -97,7 +97,7 @@ def default_services():
    "nginx": {
      "autostart": "true"
    },
-    "prometheus-aggregator": {
+    "pushgateway": {
      "autostart": "true"
    },
    "servicekey": {
--- a/conf/init/test/test_supervisord_conf_create.py
+++ b/conf/init/test/test_supervisord_conf_create.py
@ -1,8 +1,7 @@
 import os
-import pytest
-import json
-import yaml
+
 import jinja2
+import pytest

 from ..supervisord_conf_create import QUAYCONF_DIR, default_services, limit_services

@ -358,8 +357,8 @@ stderr_logfile_maxbytes=0
 stdout_events_enabled = true
 stderr_events_enabled = true

-[program:prometheus-aggregator]
-command=/usr/local/bin/prometheus-aggregator
+[program:pushgateway]
+command=/usr/local/bin/pushgateway
 autostart = true
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@ -396,7 +395,7 @@ stderr_events_enabled = true

 def test_supervisord_conf_create_all_overrides():
  config = default_services()
-  limit_services(config, "servicekey,prometheus-aggregator")
+  limit_services(config, "servicekey,pushgateway")
  rendered = render_supervisord_conf(config)

  expected = """[supervisord]
@ -741,8 +740,8 @@ stderr_logfile_maxbytes=0
 stdout_events_enabled = true
 stderr_events_enabled = true

-[program:prometheus-aggregator]
-command=/usr/local/bin/prometheus-aggregator
+[program:pushgateway]
+command=/usr/local/bin/pushgateway
 autostart = true
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
--- a/conf/supervisord.conf.jnj
+++ b/conf/supervisord.conf.jnj
@ -340,9 +340,9 @@ stderr_logfile_maxbytes=0
 stdout_events_enabled = true
 stderr_events_enabled = true

-[program:prometheus-aggregator]
-command=/usr/local/bin/prometheus-aggregator
-autostart = {{ config['prometheus-aggregator']['autostart'] }}
+[program:pushgateway]
+command=/usr/local/bin/pushgateway
+autostart = {{ config['pushgateway']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stdout
@ -373,4 +373,4 @@ stderr_logfile=/dev/stdout
 stderr_logfile_maxbytes=0
 stdout_events_enabled = true
 stderr_events_enabled = true
-# EOF NO NEWLINE
+# EOF NO NEWLINE
--- a/config.py
+++ b/config.py
@ -460,8 +460,8 @@ class DefaultConfig(ImmutableConfig):
  # The whitelist of client IDs for OAuth applications that allow for direct login.
  DIRECT_OAUTH_CLIENTID_WHITELIST = []

-  # URL that specifies the location of the prometheus stats aggregator.
-  PROMETHEUS_AGGREGATOR_URL = 'http://localhost:9092'
+  # URL that specifies the location of the prometheus pushgateway.
+  PROMETHEUS_PUSHGATEWAY_URL = 'http://localhost:9091'

  # Namespace prefix for all prometheus metrics.
  PROMETHEUS_NAMESPACE = 'quay'
--- a/local-docker.sh
+++ b/local-docker.sh
@ -38,11 +38,6 @@ initdb)
 fulldbtest)
 	d bash /src/quay/test/fulldbtest.sh
 	;;
-prom)
-	R=quay.io/quay/prom-monitor
-	docker build -t $R prom_aggregator
-	docker run --rm -it --net=host $R -loglevel=debug
-	;;
 *)
 	echo "unknown option"
 	exit 1
--- a/quay-base.dockerfile
+++ b/quay-base.dockerfile
@ -2,6 +2,8 @@

 FROM phusion/baseimage:0.10.0

+ENV OS linux
+ENV ARCH amd64
 ENV DEBIAN_FRONTEND noninteractive
 ENV HOME /root
 ENV QUAYDIR /quay-registry
@ -70,12 +72,17 @@ RUN curl -O https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz && \
    rm -rf /gocode && rm -rf /usr/local/go

 # Install jwtproxy
-RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \
-    && chmod +x /usr/local/bin/jwtproxy
+ENV JWTPROXY_VERSION=0.0.3
+RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v$(JWTPROXY_VERSION)/jwtproxy-$(OS)-$(ARCH) && \
+    chmod +x /usr/local/bin/jwtproxy

-# Install prometheus-aggregator
-RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \
-    && chmod +x /usr/local/bin/prometheus-aggregator
+# Install pushgateway
+ENV PUSHGATEWAY_VERSION=1.0.0
+RUN curl -fsSL https://github.com/prometheus/pushgateway/releases/download/$(PUSHGATEWAY_VERSION)/pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH).tar.gz | \
+    tar xz pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH)/pushgateway && \
+    mv pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH)/pushgateway /usr/local/bin/pushgateway && \
+    rm -rf pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH) && \
+    chmod +x /usr/local/bin/pushgateway

 # Install python dependencies
 COPY requirements.txt requirements-tests.txt ./
--- a/quay-entrypoint.sh
+++ b/quay-entrypoint.sh
@ -75,9 +75,9 @@ case "$QUAYENTRY" in
        echo "Entering repository mirroring mode"
        if [ -z "${QUAY_SERVICES}" ]
        then
-            export QUAY_SERVICES=repomirrorworker,prometheus-aggregator
+            export QUAY_SERVICES=repomirrorworker,pushgateway
        else
-            export QUAY_SERVICES=${QUAY_SERVICES},repomirrorworker,prometheus-aggregator
+            export QUAY_SERVICES=${QUAY_SERVICES},repomirrorworker,pushgateway
        fi
        ;&
    "registry")
--- a/test/testconfig.py
+++ b/test/testconfig.py
@ -78,7 +78,7 @@ class TestConfig(DefaultConfig):
  INSTANCE_SERVICE_KEY_KID_LOCATION = 'test/data/test.kid'
  INSTANCE_SERVICE_KEY_LOCATION = 'test/data/test.pem'

-  PROMETHEUS_AGGREGATOR_URL = None
+  PROMETHEUS_PUSHGATEWAY_URL = None

  GITHUB_LOGIN_CONFIG = {}
  GOOGLE_LOGIN_CONFIG = {}
--- a/util/config/schema.py
+++ b/util/config/schema.py
@ -62,7 +62,7 @@ INTERNAL_ONLY_PROPERTIES = {
  'TUF_GUN_PREFIX',
  'LOGGING_LEVEL',
  'SIGNED_GRANT_EXPIRATION_SEC',
-  'PROMETHEUS_AGGREGATOR_URL',
+  'PROMETHEUS_PUSHGATEWAY_URL',
  'DB_TRANSACTION_FACTORY',
  'NOTIFICATION_SEND_TIMEOUT',
  'QUEUE_METRICS_TYPE',
--- a/util/metrics/prometheus.py
+++ b/util/metrics/prometheus.py
@ -48,10 +48,10 @@ class ThreadPusher(threading.Thread):
    self._app = app

  def run(self):
-    agg_url = self._app.config.get('PROMETHEUS_AGGREGATOR_URL')
+    agg_url = self._app.config.get('PROMETHEUS_PUSHGATEWAY_URL')
    while True:
      if agg_url is None:
-        # Practically disable this worker, if there is no aggregator.
+        # Practically disable this worker, if there is no pushgateway.
        time.sleep(ONE_DAY_IN_SECONDS)
        continue

--- a/workers/globalpromstats/globalpromstats.py
+++ b/workers/globalpromstats/globalpromstats.py
@ -52,7 +52,7 @@ class GlobalPrometheusStatsWorker(Worker):
 def main():
  logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False)

-  if not app.config.get('PROMETHEUS_AGGREGATOR_URL'):
+  if not app.config.get('PROMETHEUS_PUSHGATEWAY_URL'):
    logger.debug('Prometheus not enabled; skipping global stats reporting')
    while True:
      time.sleep(100000)