vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

conf: replace prometheus aggregator w/ pushgateway

Browse source 
This change replaces the homegrown Prometheus aggregation process that
runs inside the container with the upstream Prometheus PushGateway.
This commit is contained in:
Jimmy Zelinskie 2019-11-13 15:11:59 -05:00
parent 4bf4ce33c9
commit fa1deff1ad
17 changed files with 70 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
Dockerfile
View file

@ -1,7 +1,9 @@
FROM centos:7 FROM centos:7
LABEL maintainer "thomasmckay@redhat.com" LABEL maintainer "thomasmckay@redhat.com"
ENV PYTHON_VERSION=2.7 \ ENV OS=linux \
ARCH=amd64 \
PYTHON_VERSION=2.7 \
PATH=$HOME/.local/bin/:$PATH \ PATH=$HOME/.local/bin/:$PATH \
PYTHONUNBUFFERED=1 \ PYTHONUNBUFFERED=1 \
PYTHONIOENCODING=UTF-8 \ PYTHONIOENCODING=UTF-8 \
@ -76,13 +78,18 @@ RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \
# TODO: Build jwtproxy in dist-git # TODO: Build jwtproxy in dist-git
# https://jira.coreos.com/browse/QUAY-1315 # https://jira.coreos.com/browse/QUAY-1315
RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \ ENV JWTPROXY_VERSION=0.0.3
RUN curl -fsSL -o /usr/local/bin/jwtproxy "https://github.com/coreos/jwtproxy/releases/download/v${JWTPROXY_VERSION}/jwtproxy-${OS}-${ARCH}" && \
chmod +x /usr/local/bin/jwtproxy chmod +x /usr/local/bin/jwtproxy
# TODO: Build prometheus-aggregator in dist-git # TODO: Build pushgateway in dist-git
# https://jira.coreos.com/browse/QUAY-1324 # https://jira.coreos.com/browse/QUAY-1324
RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\ ENV PUSHGATEWAY_VERSION=1.0.0
chmod +x /usr/local/bin/prometheus-aggregator RUN curl -fsSL "https://github.com/prometheus/pushgateway/releases/download/v${PUSHGATEWAY_VERSION}/pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}.tar.gz" | \
tar xz "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" && \
mv "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" /usr/local/bin/pushgateway && \
rm -rf "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}" && \
chmod +x /usr/local/bin/pushgateway
# Update local copy of AWS IP Ranges. # Update local copy of AWS IP Ranges.
RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
@ -105,7 +112,7 @@ RUN UNINSTALL_PKGS="\
yum clean all && \ yum clean all && \
rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache
EXPOSE 8080 8443 7443 EXPOSE 8080 8443 7443 9091
RUN chgrp -R 0 $QUAYDIR && \ RUN chgrp -R 0 $QUAYDIR && \
chmod -R g=u $QUAYDIR chmod -R g=u $QUAYDIR

19
Dockerfile.rhel7
View file

@ -1,7 +1,9 @@
FROM registry.redhat.io/rhel7:7.7 FROM registry.redhat.io/rhel7:7.7
LABEL maintainer "thomasmckay@redhat.com" LABEL maintainer "thomasmckay@redhat.com"
ENV PYTHON_VERSION=2.7 \ ENV OS=linux \
ARCH=amd64 \
PYTHON_VERSION=2.7 \
PATH=$HOME/.local/bin/:$PATH \ PATH=$HOME/.local/bin/:$PATH \
PYTHONUNBUFFERED=1 \ PYTHONUNBUFFERED=1 \
PYTHONIOENCODING=UTF-8 \ PYTHONIOENCODING=UTF-8 \
@ -81,13 +83,18 @@ RUN curl --silent --location https://rpm.nodesource.com/setup_8.x | bash - && \
# TODO: Build jwtproxy in dist-git # TODO: Build jwtproxy in dist-git
# https://jira.coreos.com/browse/QUAY-1315 # https://jira.coreos.com/browse/QUAY-1315
RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.3/jwtproxy-linux-x64 && \ ENV JWTPROXY_VERSION=0.0.3
RUN curl -fsSL -o /usr/local/bin/jwtproxy "https://github.com/coreos/jwtproxy/releases/download/v${JWTPROXY_VERSION}/jwtproxy-${OS}-${ARCH}" && \
chmod +x /usr/local/bin/jwtproxy chmod +x /usr/local/bin/jwtproxy
# TODO: Build prometheus-aggregator in dist-git # TODO: Build pushgateway in dist-git
# https://jira.coreos.com/browse/QUAY-1324 # https://jira.coreos.com/browse/QUAY-1324
RUN curl -fsSL -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator &&\ ENV PUSHGATEWAY_VERSION=1.0.0
chmod +x /usr/local/bin/prometheus-aggregator RUN curl -fsSL "https://github.com/prometheus/pushgateway/releases/download/v${PUSHGATEWAY_VERSION}/pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}.tar.gz" | \
tar xz "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" && \
mv "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}/pushgateway" /usr/local/bin/pushgateway && \
rm -rf "pushgateway-${PUSHGATEWAY_VERSION}.${OS}-${ARCH}" && \
chmod +x /usr/local/bin/pushgateway
# Update local copy of AWS IP Ranges. # Update local copy of AWS IP Ranges.
RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json RUN curl -fsSL https://ip-ranges.amazonaws.com/ip-ranges.json -o util/ipresolver/aws-ip-ranges.json
@ -110,7 +117,7 @@ RUN UNINSTALL_PKGS="\
yum clean all && \ yum clean all && \
rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache rm -rf /var/cache/yum /tmp/* /var/tmp/* /root/.cache
EXPOSE 8080 8443 7443 EXPOSE 8080 8443 7443 9091
RUN chgrp -R 0 $QUAYDIR && \ RUN chgrp -R 0 $QUAYDIR && \
chmod -R g=u $QUAYDIR chmod -R g=u $QUAYDIR

4
conf/init/service/interactive/prometheus-aggregator/log/run
View file

@ -1,4 +0,0 @@
#!/bin/sh
# Start the logger
exec logger -i -t prometheus-aggregator

7
conf/init/service/interactive/prometheus-aggregator/run
View file

@ -1,7 +0,0 @@
#! /bin/bash
echo 'Starting prometheus aggregator'
/usr/local/bin/prometheus-aggregator
echo 'Prometheus aggregator exited'

4
conf/init/service/interactive/pushgateway/log/run Executable file
View file

@ -0,0 +1,4 @@
#!/bin/sh
# Start the logger
exec logger -i -t pushgateway

7
conf/init/service/interactive/pushgateway/run Executable file
View file

@ -0,0 +1,7 @@
#! /bin/bash
echo 'Starting prometheus pushgateway'
/usr/local/bin/pushgateway
echo 'Prometheus pushgateway exited'

2
conf/init/supervisord_conf_create.py
View file

@ -97,7 +97,7 @@ def default_services():
"nginx": { "nginx": {
"autostart": "true" "autostart": "true"
}, },
"prometheus-aggregator": { "pushgateway": {
"autostart": "true" "autostart": "true"
}, },
"servicekey": { "servicekey": {

15
conf/init/test/test_supervisord_conf_create.py
View file

@ -1,8 +1,7 @@
import os import os
import pytest
import json
import yaml
import jinja2 import jinja2
import pytest
from ..supervisord_conf_create import QUAYCONF_DIR, default_services, limit_services from ..supervisord_conf_create import QUAYCONF_DIR, default_services, limit_services
@ -358,8 +357,8 @@ stderr_logfile_maxbytes=0
stdout_events_enabled = true stdout_events_enabled = true
stderr_events_enabled = true stderr_events_enabled = true
[program:prometheus-aggregator] [program:pushgateway]
command=/usr/local/bin/prometheus-aggregator command=/usr/local/bin/pushgateway
autostart = true autostart = true
stdout_logfile=/dev/stdout stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0 stdout_logfile_maxbytes=0
@ -396,7 +395,7 @@ stderr_events_enabled = true
def test_supervisord_conf_create_all_overrides(): def test_supervisord_conf_create_all_overrides():
config = default_services() config = default_services()
limit_services(config, "servicekey,prometheus-aggregator") limit_services(config, "servicekey,pushgateway")
rendered = render_supervisord_conf(config) rendered = render_supervisord_conf(config)
expected = """[supervisord] expected = """[supervisord]
@ -741,8 +740,8 @@ stderr_logfile_maxbytes=0
stdout_events_enabled = true stdout_events_enabled = true
stderr_events_enabled = true stderr_events_enabled = true
[program:prometheus-aggregator] [program:pushgateway]
command=/usr/local/bin/prometheus-aggregator command=/usr/local/bin/pushgateway
autostart = true autostart = true
stdout_logfile=/dev/stdout stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0 stdout_logfile_maxbytes=0

8
conf/supervisord.conf.jnj
View file

@ -340,9 +340,9 @@ stderr_logfile_maxbytes=0
stdout_events_enabled = true stdout_events_enabled = true
stderr_events_enabled = true stderr_events_enabled = true
[program:prometheus-aggregator] [program:pushgateway]
command=/usr/local/bin/prometheus-aggregator command=/usr/local/bin/pushgateway
autostart = {{ config['prometheus-aggregator']['autostart'] }} autostart = {{ config['pushgateway']['autostart'] }}
stdout_logfile=/dev/stdout stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0 stdout_logfile_maxbytes=0
stderr_logfile=/dev/stdout stderr_logfile=/dev/stdout
@ -373,4 +373,4 @@ stderr_logfile=/dev/stdout
stderr_logfile_maxbytes=0 stderr_logfile_maxbytes=0
stdout_events_enabled = true stdout_events_enabled = true
stderr_events_enabled = true stderr_events_enabled = true
# EOF NO NEWLINE # EOF NO NEWLINE

4
config.py
View file

@ -460,8 +460,8 @@ class DefaultConfig(ImmutableConfig):
# The whitelist of client IDs for OAuth applications that allow for direct login. # The whitelist of client IDs for OAuth applications that allow for direct login.
DIRECT_OAUTH_CLIENTID_WHITELIST = [] DIRECT_OAUTH_CLIENTID_WHITELIST = []
# URL that specifies the location of the prometheus stats aggregator. # URL that specifies the location of the prometheus pushgateway.
PROMETHEUS_AGGREGATOR_URL = 'http://localhost:9092' PROMETHEUS_PUSHGATEWAY_URL = 'http://localhost:9091'
# Namespace prefix for all prometheus metrics. # Namespace prefix for all prometheus metrics.
PROMETHEUS_NAMESPACE = 'quay' PROMETHEUS_NAMESPACE = 'quay'

5
local-docker.sh
View file

@ -38,11 +38,6 @@ initdb)
fulldbtest) fulldbtest)
d bash /src/quay/test/fulldbtest.sh d bash /src/quay/test/fulldbtest.sh
;; ;;
prom)
R=quay.io/quay/prom-monitor
docker build -t $R prom_aggregator
docker run --rm -it --net=host $R -loglevel=debug
;;
*) *)
echo "unknown option" echo "unknown option"
exit 1 exit 1

17
quay-base.dockerfile
View file

@ -2,6 +2,8 @@
FROM phusion/baseimage:0.10.0 FROM phusion/baseimage:0.10.0
ENV OS linux
ENV ARCH amd64
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ENV HOME /root ENV HOME /root
ENV QUAYDIR /quay-registry ENV QUAYDIR /quay-registry
@ -70,12 +72,17 @@ RUN curl -O https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz && \
rm -rf /gocode && rm -rf /usr/local/go rm -rf /gocode && rm -rf /usr/local/go
# Install jwtproxy # Install jwtproxy
RUN curl -L -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v0.0.1/jwtproxy-linux-x64 \ ENV JWTPROXY_VERSION=0.0.3
&& chmod +x /usr/local/bin/jwtproxy RUN curl -fsSL -o /usr/local/bin/jwtproxy https://github.com/coreos/jwtproxy/releases/download/v$(JWTPROXY_VERSION)/jwtproxy-$(OS)-$(ARCH) && \
chmod +x /usr/local/bin/jwtproxy
# Install prometheus-aggregator # Install pushgateway
RUN curl -L -o /usr/local/bin/prometheus-aggregator https://github.com/coreos/prometheus-aggregator/releases/download/v0.0.1-alpha/prometheus-aggregator \ ENV PUSHGATEWAY_VERSION=1.0.0
&& chmod +x /usr/local/bin/prometheus-aggregator RUN curl -fsSL https://github.com/prometheus/pushgateway/releases/download/$(PUSHGATEWAY_VERSION)/pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH).tar.gz | \
tar xz pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH)/pushgateway && \
mv pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH)/pushgateway /usr/local/bin/pushgateway && \
rm -rf pushgateway-$(PUSHGATEWAY_VERSION).$(OS)-$(ARCH) && \
chmod +x /usr/local/bin/pushgateway
# Install python dependencies # Install python dependencies
COPY requirements.txt requirements-tests.txt ./ COPY requirements.txt requirements-tests.txt ./

4
quay-entrypoint.sh
View file

@ -75,9 +75,9 @@ case "$QUAYENTRY" in
echo "Entering repository mirroring mode" echo "Entering repository mirroring mode"
if [ -z "${QUAY_SERVICES}" ] if [ -z "${QUAY_SERVICES}" ]
then then
export QUAY_SERVICES=repomirrorworker,prometheus-aggregator export QUAY_SERVICES=repomirrorworker,pushgateway
else else
export QUAY_SERVICES=${QUAY_SERVICES},repomirrorworker,prometheus-aggregator export QUAY_SERVICES=${QUAY_SERVICES},repomirrorworker,pushgateway
fi fi
;& ;&
"registry") "registry")

2
test/testconfig.py
View file

@ -78,7 +78,7 @@ class TestConfig(DefaultConfig):
INSTANCE_SERVICE_KEY_KID_LOCATION = 'test/data/test.kid' INSTANCE_SERVICE_KEY_KID_LOCATION = 'test/data/test.kid'
INSTANCE_SERVICE_KEY_LOCATION = 'test/data/test.pem' INSTANCE_SERVICE_KEY_LOCATION = 'test/data/test.pem'
PROMETHEUS_AGGREGATOR_URL = None PROMETHEUS_PUSHGATEWAY_URL = None
GITHUB_LOGIN_CONFIG = {} GITHUB_LOGIN_CONFIG = {}
GOOGLE_LOGIN_CONFIG = {} GOOGLE_LOGIN_CONFIG = {}

2
util/config/schema.py
View file

@ -62,7 +62,7 @@ INTERNAL_ONLY_PROPERTIES = {
'TUF_GUN_PREFIX', 'TUF_GUN_PREFIX',
'LOGGING_LEVEL', 'LOGGING_LEVEL',
'SIGNED_GRANT_EXPIRATION_SEC', 'SIGNED_GRANT_EXPIRATION_SEC',
'PROMETHEUS_AGGREGATOR_URL', 'PROMETHEUS_PUSHGATEWAY_URL',
'DB_TRANSACTION_FACTORY', 'DB_TRANSACTION_FACTORY',
'NOTIFICATION_SEND_TIMEOUT', 'NOTIFICATION_SEND_TIMEOUT',
'QUEUE_METRICS_TYPE', 'QUEUE_METRICS_TYPE',

4
util/metrics/prometheus.py
View file

@ -48,10 +48,10 @@ class ThreadPusher(threading.Thread):
self._app = app self._app = app
def run(self): def run(self):
agg_url = self._app.config.get('PROMETHEUS_AGGREGATOR_URL') agg_url = self._app.config.get('PROMETHEUS_PUSHGATEWAY_URL')
while True: while True:
if agg_url is None: if agg_url is None:
# Practically disable this worker, if there is no aggregator. # Practically disable this worker, if there is no pushgateway.
time.sleep(ONE_DAY_IN_SECONDS) time.sleep(ONE_DAY_IN_SECONDS)
continue continue

2
workers/globalpromstats/globalpromstats.py
View file

@ -52,7 +52,7 @@ class GlobalPrometheusStatsWorker(Worker):
def main(): def main():
logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False) logging.config.fileConfig(logfile_path(debug=False), disable_existing_loggers=False)
if not app.config.get('PROMETHEUS_AGGREGATOR_URL'): if not app.config.get('PROMETHEUS_PUSHGATEWAY_URL'):
logger.debug('Prometheus not enabled; skipping global stats reporting') logger.debug('Prometheus not enabled; skipping global stats reporting')
while True: while True:
time.sleep(100000) time.sleep(100000)