Send everything in production through the gunicorn logger (allows rotation using USR1). Add a gunicorn logger that emits the logstash format. Move the gunicorn config to the conf subdir. Update the postrotate script to hopefully work.

README.md

@@ -21,7 +21,7 @@ running:
 
 ```
 sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf
-sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application
+sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c conf/gunicorn_config.py application:application
 ```
 
 start the log shipper:

conf/gunicorn_config.py

@@ -0,0 +1,10 @@
+bind = 'unix:/tmp/gunicorn.sock'
+workers = 8
+worker_class = 'gevent'
+timeout = 2000
+daemon = True
+pidfile = '/mnt/logs/gunicorn.pid'
+errorlog = '/mnt/logs/application.log'
+loglevel = 'debug'
+logger_class = 'util.glogger.LogstashLogger'
+pythonpath = '.'

conf/gunicorn_local.py

@@ -0,0 +1,9 @@
+bind = '0.0.0.0:5000'
+workers = 2
+worker_class = 'gevent'
+timeout = 2000
+daemon = False
+errorlog = '-'
+loglevel = 'debug'
+logger_class = 'util.glogger.LogstashLogger'
+pythonpath = '.'

conf/logrotate/quay-logrotate

@@ -8,7 +8,7 @@
   create 644 root root
 
   postrotate
-    [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid`
+    kill -USR1 `cat /mnt/logs/nginx.pid`
   endscript
 }
 
@@ -22,7 +22,7 @@
   create 644 root root
 
   postrotate
-    [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid`
+    kill -USR1 `cat /mnt/logs/nginx.pid`
   endscript
 }
 
@@ -36,6 +36,6 @@
   create 644 ubuntu ubuntu
 
   postrotate
-    [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/gunicorn.pid`
+    kill -USR1 `cat /mnt/logs/gunicorn.pid`
   endscript
 }

config.py

@@ -135,14 +135,10 @@ class BuildNodeConfig(object):
   BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
 
 
-def logs_init_builder(level=logging.DEBUG, logfile=None):
+def logs_init_builder(level=logging.DEBUG):
   @staticmethod
   def init_logs():
-    if logfile:
-      handler = logging.FileHandler(logfile)
-    else:
-      handler = logging.StreamHandler()
-
+    handler = logging.StreamHandler()
     root_logger = logging.getLogger('')
     root_logger.setLevel(level)
     formatter = logstash_formatter.LogstashFormatter()
@@ -182,5 +178,5 @@ class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL,
                        GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig,
                        S3Userfiles):
 
-  LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log')
+  LOGGING_CONFIG = logs_init_builder()
   SEND_FILE_MAX_AGE_DEFAULT = 0

gunicorn_config.py

@@ -1,6 +0,0 @@
-bind = 'unix:/tmp/gunicorn.sock'
-workers = 8
-worker_class = 'gevent'
-timeout = 2000
-daemon = True
-pidfile = '/mnt/logs/gunicorn.pid'

util/glogger.py

@@ -0,0 +1,23 @@
+import logging
+import logstash_formatter
+import gunicorn.glogging
+
+from gunicorn import util
+
+class LogstashLogger(gunicorn.glogging.Logger):
+  def _set_handler(self, log, output, fmt):
+    # remove previous gunicorn log handler
+    h = self._get_gunicorn_handler(log)
+    if h:
+      log.handlers.remove(h)
+
+    if output is not None:
+      if output == "-":
+        h = logging.StreamHandler()
+      else:
+        util.check_is_writeable(output)
+        h = logging.FileHandler(output)
+
+      h.setFormatter(logstash_formatter.LogstashFormatter())
+      h._gunicorn = True
+      log.addHandler(h)