From fa1deff1adea487d3a30b4fc683311278e379c83 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 13 Nov 2019 15:11:59 -0500 Subject: [PATCH] conf: replace prometheus aggregator w/ pushgateway This change replaces the homegrown Prometheus aggregation process that runs inside the container with the upstream Prometheus PushGateway. --- Dockerfile | 19 +++++++++++++------ Dockerfile.rhel7 | 19 +++++++++++++------ .../interactive/prometheus-aggregator/log/run | 4 ---- .../interactive/prometheus-aggregator/run | 7 ------- .../service/interactive/pushgateway/log/run | 4 ++++ conf/init/service/interactive/pushgateway/run | 7 +++++++ conf/init/supervisord_conf_create.py | 2 +- .../init/test/test_supervisord_conf_create.py | 15 +++++++-------- conf/supervisord.conf.jnj | 8 ++++---- config.py | 4 ++-- local-docker.sh | 5 ----- quay-base.dockerfile | 17 ++++++++++++----- quay-entrypoint.sh | 4 ++-- test/testconfig.py | 2 +- util/config/schema.py | 2 +- util/metrics/prometheus.py | 4 ++-- workers/globalpromstats/globalpromstats.py | 2 +- 17 files changed, 70 insertions(+), 55 deletions(-) delete mode 100755 conf/init/service/interactive/prometheus-aggregator/log/run delete mode 100755 conf/init/service/interactive/prometheus-aggregator/run create mode 100755 conf/init/service/interactive/pushgateway/log/run create mode 100755 conf/init/service/interactive/pushgateway/run diff --git a/Dockerfile b/Dockerfile index 36324c569..a9a75fffa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ FROM centos:7 LABEL maintainer "thomasmckay@redhat.com" -ENV PYTHON_VERSION=2.7 \ +ENV OS=linux \ + ARCH=amd64 \ + PYTHON_VERSION=2.7 \ PATH=$HOME/.local/bin/:$PATH \ PYTHONUNBUFFERED=1 \ PYTHONIOENCODING=UTF-8 \ @@ -76,13 +78,18 @@ RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \ # TODO: Build jwtproxy in dist-git # https://jira.coreos.com/browse/QUAY-1315 -RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \ +ENV JWTPROXY_VERSION=0.0.3 +RUN curl -fsSL -o /usr/local/bin/jwtproxy "https://github.com/coreos/jwtproxy/releases/download/v${JWTPROXY_VERSION}/jwtproxy-${OS}-${ARCH}" && \ chmod +x /usr/local/bin/jwtproxy -# TODO: Build prometheus-aggregator in dist-git +# TODO: Build pushgateway in dist-git # https://jira.coreos.com/browse/QUAY-1324 -RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\ - chmod +x /usr/local/bin/prometheus-aggregator +ENV PUSHGATEWAY_VERSION=1.0.0 +RUN curl -fsSL "https://github.com/prometheus/pushgateway/releases/download/v${PUSHGATEWAY_VERSION}/pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}.tar.gz" | \ + tar xz "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" && \ + mv "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" /usr/local/bin/pushgateway && \ + rm -rf "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}" && \ + chmod +x /usr/local/bin/pushgateway # Update local copy of AWS IP Ranges. RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json @@ -105,7 +112,7 @@ RUN UNINSTALL_PKGS="\ yum clean all && \ rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache -EXPOSE 8080 8443 7443 +EXPOSE 8080 8443 7443 9091 RUN chgrp -R 0 $QUAYDIR && \ chmod -R g=u $QUAYDIR diff --git a/Dockerfile.rhel7 b/Dockerfile.rhel7 index c52822462..249b898bb 100644 --- a/Dockerfile.rhel7 +++ b/Dockerfile.rhel7 @@ -1,7 +1,9 @@ FROM registry.redhat.io/rhel7:7.7 LABEL maintainer "thomasmckay@redhat.com" -ENV PYTHON_VERSION=2.7 \ +ENV OS=linux \ + ARCH=amd64 \ + PYTHON_VERSION=2.7 \ PATH=$HOME/.local/bin/:$PATH \ PYTHONUNBUFFERED=1 \ PYTHONIOENCODING=UTF-8 \ @@ -81,13 +83,18 @@ RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \ # TODO: Build jwtproxy in dist-git # https://jira.coreos.com/browse/QUAY-1315 -RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \ +ENV JWTPROXY_VERSION=0.0.3 +RUN curl -fsSL -o /usr/local/bin/jwtproxy "https://github.com/coreos/jwtproxy/releases/download/v${JWTPROXY_VERSION}/jwtproxy-${OS}-${ARCH}" && \ chmod +x /usr/local/bin/jwtproxy -# TODO: Build prometheus-aggregator in dist-git +# TODO: Build pushgateway in dist-git # https://jira.coreos.com/browse/QUAY-1324 -RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\ - chmod +x /usr/local/bin/prometheus-aggregator +ENV PUSHGATEWAY_VERSION=1.0.0 +RUN curl -fsSL "https://github.com/prometheus/pushgateway/releases/download/v${PUSHGATEWAY_VERSION}/pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}.tar.gz" | \ + tar xz "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" && \ + mv "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" /usr/local/bin/pushgateway && \ + rm -rf "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}" && \ + chmod +x /usr/local/bin/pushgateway # Update local copy of AWS IP Ranges. RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json @@ -110,7 +117,7 @@ RUN UNINSTALL_PKGS="\ yum clean all && \ rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache -EXPOSE 8080 8443 7443 +EXPOSE 8080 8443 7443 9091 RUN chgrp -R 0 $QUAYDIR && \ chmod -R g=u $QUAYDIR diff --git a/conf/init/service/interactive/prometheus-aggregator/log/run b/conf/init/service/interactive/prometheus-aggregator/log/run deleted file mode 100755 index a1ca97fa3..000000000 --- a/conf/init/service/interactive/prometheus-aggregator/log/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# Start the logger -exec logger -i -t prometheus-aggregator diff --git a/conf/init/service/interactive/prometheus-aggregator/run b/conf/init/service/interactive/prometheus-aggregator/run deleted file mode 100755 index fc9b157c7..000000000 --- a/conf/init/service/interactive/prometheus-aggregator/run +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/bash - -echo 'Starting prometheus aggregator' - -/usr/local/bin/prometheus-aggregator - -echo 'Prometheus aggregator exited' \ No newline at end of file diff --git a/conf/init/service/interactive/pushgateway/log/run b/conf/init/service/interactive/pushgateway/log/run new file mode 100755 index 000000000..8c69a4ac6 --- /dev/null +++ b/conf/init/service/interactive/pushgateway/log/run @@ -0,0 +1,4 @@ +#!/bin/sh + +# Start the logger +exec logger -i -t pushgateway diff --git a/conf/init/service/interactive/pushgateway/run b/conf/init/service/interactive/pushgateway/run new file mode 100755 index 000000000..9e51941ef --- /dev/null +++ b/conf/init/service/interactive/pushgateway/run @@ -0,0 +1,7 @@ +#! /bin/bash + +echo 'Starting prometheus pushgateway' + +/usr/local/bin/pushgateway + +echo 'Prometheus pushgateway exited' diff --git a/conf/init/supervisord_conf_create.py b/conf/init/supervisord_conf_create.py index 50f5cabbf..4bf55c23d 100644 --- a/conf/init/supervisord_conf_create.py +++ b/conf/init/supervisord_conf_create.py @@ -97,7 +97,7 @@ def default_services(): "nginx": { "autostart": "true" }, - "prometheus-aggregator": { + "pushgateway": { "autostart": "true" }, "servicekey": { diff --git a/conf/init/test/test_supervisord_conf_create.py b/conf/init/test/test_supervisord_conf_create.py index 8972b2e39..0161b00c6 100644 --- a/conf/init/test/test_supervisord_conf_create.py +++ b/conf/init/test/test_supervisord_conf_create.py @@ -1,8 +1,7 @@ import os -import pytest -import json -import yaml + import jinja2 +import pytest from ..supervisord_conf_create import QUAYCONF_DIR, default_services, limit_services @@ -358,8 +357,8 @@ stderr_logfile_maxbytes=0 stdout_events_enabled = true stderr_events_enabled = true -[program:prometheus-aggregator] -command=/usr/local/bin/prometheus-aggregator +[program:pushgateway] +command=/usr/local/bin/pushgateway autostart = true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 @@ -396,7 +395,7 @@ stderr_events_enabled = true def test_supervisord_conf_create_all_overrides(): config = default_services() - limit_services(config, "servicekey,prometheus-aggregator") + limit_services(config, "servicekey,pushgateway") rendered = render_supervisord_conf(config) expected = """[supervisord] @@ -741,8 +740,8 @@ stderr_logfile_maxbytes=0 stdout_events_enabled = true stderr_events_enabled = true -[program:prometheus-aggregator] -command=/usr/local/bin/prometheus-aggregator +[program:pushgateway] +command=/usr/local/bin/pushgateway autostart = true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 diff --git a/conf/supervisord.conf.jnj b/conf/supervisord.conf.jnj index b5224250d..08237c7a4 100644 --- a/conf/supervisord.conf.jnj +++ b/conf/supervisord.conf.jnj @@ -340,9 +340,9 @@ stderr_logfile_maxbytes=0 stdout_events_enabled = true stderr_events_enabled = true -[program:prometheus-aggregator] -command=/usr/local/bin/prometheus-aggregator -autostart = {{ config['prometheus-aggregator']['autostart'] }} +[program:pushgateway] +command=/usr/local/bin/pushgateway +autostart = {{ config['pushgateway']['autostart'] }} stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stdout @@ -373,4 +373,4 @@ stderr_logfile=/dev/stdout stderr_logfile_maxbytes=0 stdout_events_enabled = true stderr_events_enabled = true -# EOF NO NEWLINE \ No newline at end of file +# EOF NO NEWLINE diff --git a/config.py b/config.py index f0743c6af..92e789fa9 100644 --- a/config.py +++ b/config.py @@ -460,8 +460,8 @@ class DefaultConfig(ImmutableConfig): # The whitelist of client IDs for OAuth applications that allow for direct login. DIRECT_OAUTH_CLIENTID_WHITELIST = [] - # URL that specifies the location of the prometheus stats aggregator. - PROMETHEUS_AGGREGATOR_URL = 'http://localhost:9092' + # URL that specifies the location of the prometheus pushgateway. + PROMETHEUS_PUSHGATEWAY_URL = 'http://localhost:9091' # Namespace prefix for all prometheus metrics. PROMETHEUS_NAMESPACE = 'quay' diff --git a/local-docker.sh b/local-docker.sh index 884346ec3..3f93aca80 100755 --- a/local-docker.sh +++ b/local-docker.sh @@ -38,11 +38,6 @@ initdb) fulldbtest) d bash /src/quay/test/fulldbtest.sh ;; -prom) - R=quay.io/quay/prom-monitor - docker build -t $R prom_aggregator - docker run --rm -it --net=host $R -loglevel=debug - ;; *) echo "unknown option" exit 1 diff --git a/quay-base.dockerfile b/quay-base.dockerfile index f23d12d81..a692e273d 100644 --- a/quay-base.dockerfile +++ b/quay-base.dockerfile @@ -2,6 +2,8 @@ FROM phusion/baseimage:0.10.0 +ENV OS linux +ENV ARCH amd64 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root ENV QUAYDIR /quay-registry @@ -70,12 +72,17 @@ RUN curl -O https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz && \ rm -rf /gocode && rm -rf /usr/local/go # Install jwtproxy -RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \ - && chmod +x /usr/local/bin/jwtproxy +ENV JWTPROXY_VERSION=0.0.3 +RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v$(JWTPROXY_VERSION)/jwtproxy-$(OS)-$(ARCH) && \ + chmod +x /usr/local/bin/jwtproxy -# Install prometheus-aggregator -RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \ - && chmod +x /usr/local/bin/prometheus-aggregator +# Install pushgateway +ENV PUSHGATEWAY_VERSION=1.0.0 +RUN curl -fsSL https://github.com/prometheus/pushgateway/releases/download/$(PUSHGATEWAY_VERSION)/pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH).tar.gz | \ + tar xz pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH)/pushgateway && \ + mv pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH)/pushgateway /usr/local/bin/pushgateway && \ + rm -rf pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH) && \ + chmod +x /usr/local/bin/pushgateway # Install python dependencies COPY requirements.txt requirements-tests.txt ./ diff --git a/quay-entrypoint.sh b/quay-entrypoint.sh index 59573b83a..3a761c40b 100755 --- a/quay-entrypoint.sh +++ b/quay-entrypoint.sh @@ -75,9 +75,9 @@ case "$QUAYENTRY" in echo "Entering repository mirroring mode" if [ -z "${QUAY_SERVICES}" ] then - export QUAY_SERVICES=repomirrorworker,prometheus-aggregator + export QUAY_SERVICES=repomirrorworker,pushgateway else - export QUAY_SERVICES=${QUAY_SERVICES},repomirrorworker,prometheus-aggregator + export QUAY_SERVICES=${QUAY_SERVICES},repomirrorworker,pushgateway fi ;& "registry") diff --git a/test/testconfig.py b/test/testconfig.py index 52be3c1f4..539654c54 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -78,7 +78,7 @@ class TestConfig(DefaultConfig): INSTANCE_SERVICE_KEY_KID_LOCATION = 'test/data/test.kid' INSTANCE_SERVICE_KEY_LOCATION = 'test/data/test.pem' - PROMETHEUS_AGGREGATOR_URL = None + PROMETHEUS_PUSHGATEWAY_URL = None GITHUB_LOGIN_CONFIG = {} GOOGLE_LOGIN_CONFIG = {} diff --git a/util/config/schema.py b/util/config/schema.py index 959d9d49f..e3868ce81 100644 --- a/util/config/schema.py +++ b/util/config/schema.py @@ -62,7 +62,7 @@ INTERNAL_ONLY_PROPERTIES = { 'TUF_GUN_PREFIX', 'LOGGING_LEVEL', 'SIGNED_GRANT_EXPIRATION_SEC', - 'PROMETHEUS_AGGREGATOR_URL', + 'PROMETHEUS_PUSHGATEWAY_URL', 'DB_TRANSACTION_FACTORY', 'NOTIFICATION_SEND_TIMEOUT', 'QUEUE_METRICS_TYPE', diff --git a/util/metrics/prometheus.py b/util/metrics/prometheus.py index f281bc9fe..c356a0760 100644 --- a/util/metrics/prometheus.py +++ b/util/metrics/prometheus.py @@ -48,10 +48,10 @@ class ThreadPusher(threading.Thread): self._app = app def run(self): - agg_url = self._app.config.get('PROMETHEUS_AGGREGATOR_URL') + agg_url = self._app.config.get('PROMETHEUS_PUSHGATEWAY_URL') while True: if agg_url is None: - # Practically disable this worker, if there is no aggregator. + # Practically disable this worker, if there is no pushgateway. time.sleep(ONE_DAY_IN_SECONDS) continue diff --git a/workers/globalpromstats/globalpromstats.py b/workers/globalpromstats/globalpromstats.py index dc78a146a..db0aada08 100644 --- a/workers/globalpromstats/globalpromstats.py +++ b/workers/globalpromstats/globalpromstats.py @@ -52,7 +52,7 @@ class GlobalPrometheusStatsWorker(Worker): def main(): logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False) - if not app.config.get('PROMETHEUS_AGGREGATOR_URL'): + if not app.config.get('PROMETHEUS_PUSHGATEWAY_URL'): logger.debug('Prometheus not enabled; skipping global stats reporting') while True: time.sleep(100000)