Make gunicorn worker count scale automatically and be configurable

Fixes https://jira.coreos.com/browse/QS-117

conf/gunicorn_local.py

@@ -2,13 +2,16 @@ import sys
 import os
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 
+import multiprocessing
+import logging
+
 from Crypto import Random
 from util.log import logfile_path
 
 
 logconfig = logfile_path(debug=True)
 bind = '0.0.0.0:5000'
-workers = 2
+workers = max(int(os.getenv("WORKER_COUNT_LOCAL", multiprocessing.cpu_count())), 2)
 worker_class = 'gevent'
 daemon = False
 pythonpath = '.'
@@ -18,3 +21,7 @@ def post_fork(server, worker):
   # Reset the Random library to ensure it won't raise the "PID check failed." error after
   # gunicorn forks.
   Random.atfork()
+
+def when_ready(server):
+  logger = logging.getLogger(__name__)
+  logger.debug('Starting local gunicorn with %s workers and %s worker class', workers, worker_class)

conf/gunicorn_registry.py

@@ -2,13 +2,16 @@ import sys
 import os
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 
+import multiprocessing
+import logging
+
 from Crypto import Random
 from util.log import logfile_path
 
 
 logconfig = logfile_path(debug=False)
 bind = 'unix:/tmp/gunicorn_registry.sock'
-workers = 8
+workers = max(int(os.getenv("WORKER_COUNT_REGISTRY", multiprocessing.cpu_count() * 4)), 8)
 worker_class = 'gevent'
 pythonpath = '.'
 preload_app = True
@@ -18,3 +21,8 @@ def post_fork(server, worker):
   # Reset the Random library to ensure it won't raise the "PID check failed." error after
   # gunicorn forks.
   Random.atfork()
+
+def when_ready(server):
+  logger = logging.getLogger(__name__)
+  logger.debug('Starting registry gunicorn with %s workers and %s worker class', workers,
+               worker_class)

conf/gunicorn_secscan.py

@@ -2,13 +2,16 @@ import sys
 import os
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 
+import multiprocessing
+import logging
+
 from Crypto import Random
 from util.log import logfile_path
 
 
 logconfig = logfile_path(debug=False)
 bind = 'unix:/tmp/gunicorn_secscan.sock'
-workers = 2
+workers = max(int(os.getenv("WORKER_COUNT_SECSCAN", multiprocessing.cpu_count())), 2)
 worker_class = 'gevent'
 pythonpath = '.'
 preload_app = True
@@ -18,3 +21,8 @@ def post_fork(server, worker):
   # Reset the Random library to ensure it won't raise the "PID check failed." error after
   # gunicorn forks.
   Random.atfork()
+
+def when_ready(server):
+  logger = logging.getLogger(__name__)
+  logger.debug('Starting secscan gunicorn with %s workers and %s worker class', workers,
+               worker_class)

conf/gunicorn_verbs.py

@@ -2,13 +2,16 @@ import sys
 import os
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 
+import multiprocessing
+import logging
+
 from Crypto import Random
 from util.log import logfile_path
 
 logconfig = logfile_path(debug=False)
 
 bind = 'unix:/tmp/gunicorn_verbs.sock'
-workers = 4
+workers = max(int(os.getenv("WORKER_COUNT_VERBS", multiprocessing.cpu_count())), 2)
 pythonpath = '.'
 preload_app = True
 timeout = 2000  # Because sync workers
@@ -18,3 +21,7 @@ def post_fork(server, worker):
   # Reset the Random library to ensure it won't raise the "PID check failed." error after
   # gunicorn forks.
   Random.atfork()
+
+def when_ready(server):
+  logger = logging.getLogger(__name__)
+  logger.debug('Starting verbs gunicorn with %s workers and sync worker class', workers)

conf/gunicorn_web.py

@@ -2,6 +2,9 @@ import sys
 import os
 sys.path.append(os.path.join(os.path.dirname(__file__), "../"))
 
+import multiprocessing
+import logging
+
 from Crypto import Random
 from util.log import logfile_path
 
@@ -9,7 +12,7 @@ from util.log import logfile_path
 logconfig = logfile_path(debug=False)
 
 bind = 'unix:/tmp/gunicorn_web.sock'
-workers = 2
+workers = max(int(os.getenv("WORKER_COUNT_WEB", multiprocessing.cpu_count())), 2)
 worker_class = 'gevent'
 pythonpath = '.'
 preload_app = True
@@ -18,3 +21,8 @@ def post_fork(server, worker):
   # Reset the Random library to ensure it won't raise the "PID check failed." error after
   # gunicorn forks.
   Random.atfork()
+
+def when_ready(server):
+  logger = logging.getLogger(__name__)
+  logger.debug('Starting web gunicorn with %s workers and %s worker class', workers,
+               worker_class)