From 1f420b82f61013d0bede92a781d42b830c25cf2f Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie+git@gmail.com>
Date: Fri, 15 Nov 2019 16:29:20 -0500
Subject: [PATCH] conf/init: add startup dependencies on pushgateway

Many processes should not start until the pushgateway has come online
and is ready to serve traffic. This change adds a simple loop in the run
command in order to spinlock until this condition as been met.
---
 .../interactive/prometheus-aggregator/log/run |  4 ++
 .../interactive/prometheus-aggregator/run     |  7 +++
 conf/supervisord.conf.jnj                     | 48 +++++++++----------
 util/wait_for_endpoints.py                    | 22 +++++++++
 4 files changed, 57 insertions(+), 24 deletions(-)
 create mode 100755 conf/init/service/interactive/prometheus-aggregator/log/run
 create mode 100755 conf/init/service/interactive/prometheus-aggregator/run
 create mode 100644 util/wait_for_endpoints.py

diff --git a/conf/init/service/interactive/prometheus-aggregator/log/run b/conf/init/service/interactive/prometheus-aggregator/log/run
new file mode 100755
index 000000000..a1ca97fa3
--- /dev/null
+++ b/conf/init/service/interactive/prometheus-aggregator/log/run
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# Start the logger
+exec logger -i -t prometheus-aggregator
diff --git a/conf/init/service/interactive/prometheus-aggregator/run b/conf/init/service/interactive/prometheus-aggregator/run
new file mode 100755
index 000000000..fc9b157c7
--- /dev/null
+++ b/conf/init/service/interactive/prometheus-aggregator/run
@@ -0,0 +1,7 @@
+#! /bin/bash
+
+echo 'Starting prometheus aggregator'
+
+/usr/local/bin/prometheus-aggregator
+
+echo 'Prometheus aggregator exited'
\ No newline at end of file
diff --git a/conf/supervisord.conf.jnj b/conf/supervisord.conf.jnj
index 08237c7a4..eddaa6c15 100644
--- a/conf/supervisord.conf.jnj
+++ b/conf/supervisord.conf.jnj
@@ -23,7 +23,7 @@ result_handler = supervisor_stdout:event_handler
 [program:blobuploadcleanupworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.blobuploadcleanupworker.blobuploadcleanupworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.blobuploadcleanupworker.blobuploadcleanupworker
 autostart = {{ config['blobuploadcleanupworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -35,7 +35,7 @@ stderr_events_enabled = true
 [program:buildlogsarchiver]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.buildlogsarchiver.buildlogsarchiver
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.buildlogsarchiver.buildlogsarchiver
 autostart = {{ config['buildlogsarchiver']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -47,7 +47,7 @@ stderr_events_enabled = true
 [program:builder]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m buildman.builder
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m buildman.builder
 autostart = {{ config['builder']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -59,7 +59,7 @@ stderr_events_enabled = true
 [program:chunkcleanupworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.chunkcleanupworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.chunkcleanupworker
 autostart = {{ config['chunkcleanupworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -71,7 +71,7 @@ stderr_events_enabled = true
 [program:expiredappspecifictokenworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.expiredappspecifictokenworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.expiredappspecifictokenworker
 autostart = {{ config['expiredappspecifictokenworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -95,7 +95,7 @@ stderr_events_enabled = true
 [program:gcworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.gc.gcworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.gc.gcworker
 autostart = {{ config['gcworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -107,7 +107,7 @@ stderr_events_enabled = true
 [program:globalpromstats]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.globalpromstats.globalpromstats
+command=sh -c python python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.globalpromstats.globalpromstats
 autostart = {{ config['globalpromstats']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -119,7 +119,7 @@ stderr_events_enabled = true
 [program:labelbackfillworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.labelbackfillworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.labelbackfillworker
 autostart = {{ config['labelbackfillworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -131,7 +131,7 @@ stderr_events_enabled = true
 [program:logrotateworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.logrotateworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.logrotateworker
 autostart = {{ config['logrotateworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -143,7 +143,7 @@ stderr_events_enabled = true
 [program:namespacegcworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.namespacegcworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.namespacegcworker
 autostart = {{ config['namespacegcworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -155,7 +155,7 @@ stderr_events_enabled = true
 [program:notificationworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.notificationworker.notificationworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.notificationworker.notificationworker
 autostart = {{ config['notificationworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -167,7 +167,7 @@ stderr_events_enabled = true
 [program:queuecleanupworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.queuecleanupworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.queuecleanupworker
 autostart = {{ config['queuecleanupworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -179,7 +179,7 @@ stderr_events_enabled = true
 [program:repositoryactioncounter]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.repositoryactioncounter
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.repositoryactioncounter
 autostart = {{ config['repositoryactioncounter']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -191,7 +191,7 @@ stderr_events_enabled = true
 [program:security_notification_worker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.security_notification_worker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.security_notification_worker
 autostart = {{ config['security_notification_worker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -203,7 +203,7 @@ stderr_events_enabled = true
 [program:securityworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.securityworker.securityworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.securityworker.securityworker
 autostart = {{ config['securityworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -215,7 +215,7 @@ stderr_events_enabled = true
 [program:storagereplication]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.storagereplication
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.storagereplication
 autostart = {{ config['storagereplication']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -227,7 +227,7 @@ stderr_events_enabled = true
 [program:tagbackfillworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.tagbackfillworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.tagbackfillworker
 autostart = {{ config['tagbackfillworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -239,7 +239,7 @@ stderr_events_enabled = true
 [program:teamsyncworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.teamsyncworker.teamsyncworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.teamsyncworker.teamsyncworker
 autostart = {{ config['teamsyncworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -263,7 +263,7 @@ stderr_events_enabled = true
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s,
   DB_CONNECTION_POOLING=%(ENV_DB_CONNECTION_POOLING_REGISTRY)s
-command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_registry.py registry:application
+command=sh -c python -m util/wait_for_endpoints.py http://localhost:9091/-/ready && nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_registry.py registry:application
 autostart = {{ config['gunicorn-registry']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -275,7 +275,7 @@ stderr_events_enabled = true
 [program:gunicorn-secscan]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=gunicorn -c %(ENV_QUAYCONF)s/gunicorn_secscan.py secscan:application
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && gunicorn -c %(ENV_QUAYCONF)s/gunicorn_secscan.py secscan:application
 autostart = {{ config['gunicorn-secscan']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -287,7 +287,7 @@ stderr_events_enabled = true
 [program:gunicorn-verbs]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_verbs.py verbs:application
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && nice -n 10 gunicorn -c %(ENV_QUAYCONF)s/gunicorn_verbs.py verbs:application
 autostart = {{ config['gunicorn-verbs']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -299,7 +299,7 @@ stderr_events_enabled = true
 [program:gunicorn-web]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=gunicorn -c %(ENV_QUAYCONF)s/gunicorn_web.py web:application
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && gunicorn -c %(ENV_QUAYCONF)s/gunicorn_web.py web:application
 autostart = {{ config['gunicorn-web']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -353,7 +353,7 @@ stderr_events_enabled = true
 [program:servicekey]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.servicekeyworker.servicekeyworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.servicekeyworker.servicekeyworker
 autostart = {{ config['servicekey']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -365,7 +365,7 @@ stderr_events_enabled = true
 [program:repomirrorworker]
 environment=
   PYTHONPATH=%(ENV_QUAYDIR)s
-command=python -m workers.repomirrorworker.repomirrorworker
+command=sh -c python -m util.wait_for_endpoints http://localhost:9091/-/ready && python -m workers.repomirrorworker.repomirrorworker
 autostart = {{ config['repomirrorworker']['autostart'] }}
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
diff --git a/util/wait_for_endpoints.py b/util/wait_for_endpoints.py
new file mode 100644
index 000000000..edc191956
--- /dev/null
+++ b/util/wait_for_endpoints.py
@@ -0,0 +1,22 @@
+from argparse import ArgumentParser
+from contextlib import closing
+from time import sleep
+
+import requests
+
+
+def main():
+  parser = ArgumentParser(description="block until the given endpoints return 200")
+  parser.add_argument('endpoints', type=str, nargs='+', help='the endpoints to wait for')
+  for endpoint in parser.parse_args().endpoints:
+    listening = False
+    while not listening:
+      try:
+        listening = requests.get(endpoint).status_code == 200
+      except requests.exceptions.ConnectionError:
+        pass
+      sleep(1)
+
+
+if __name__ == '__main__':
+  main()