Revamp the logging a bit. Not quite done yet.

application.py

@@ -1,10 +1,8 @@
 import logging
+import logging.config
 
 from app import app as application
 
-# Initialize logging
-application.config['LOGGING_CONFIG']()
-
 from data.model import db as model_db
 
 # Turn off debug logging for boto
@@ -41,4 +39,9 @@ def close_db(exc):
 application.teardown_request(close_db)
 
 if __name__ == '__main__':
+  logging.config.fileConfig('conf/logging_local.conf', disable_existing_loggers=False)
+
+  profile = logging.getLogger('application.profiler')
+  profile.debug('This is a profiling statement')
+
   application.run(port=5000, debug=True, threaded=True, host='0.0.0.0')

conf/gunicorn_config.py

@@ -3,7 +3,5 @@ workers = 8
 worker_class = 'gevent'
 timeout = 2000
 pidfile = '/tmp/gunicorn.pid'
-errorlog = '/mnt/logs/application.log'
+logconfig = 'conf/logging.conf'
-loglevel = 'debug'
-logger_class = 'util.glogger.LogstashLogger'
 pythonpath = '.'

conf/gunicorn_local.py

@@ -3,7 +3,5 @@ workers = 2
 worker_class = 'gevent'
 timeout = 2000
 daemon = False
-errorlog = '-'
+logconfig = 'conf/logging_local.conf'
-loglevel = 'debug'
-logger_class = 'util.glogger.LogstashLogger'
 pythonpath = '.'

conf/logging.conf

@@ -0,0 +1,39 @@
+[loggers]
+keys=root, gunicorn.error, gunicorn.access
+
+[handlers]
+keys=error_file
+
+[formatters]
+keys=generic
+
+[logger_application.profiler]
+level=DEBUG
+handlers=error_file
+propagate=0
+qualname=application.profiler
+
+[logger_root]
+level=DEBUG
+handlers=error_file
+
+[logger_gunicorn.error]
+level=INFO
+handlers=error_file
+propagate=1
+qualname=gunicorn.error
+
+[logger_gunicorn.access]
+level=INFO
+handlers=error_file
+propagate=0
+qualname=gunicorn.access
+
+[handler_error_file]
+class=logging.FileHandler
+formatter=generic
+args=('/mnt/logs/application.log',)
+
+[formatter_generic]
+format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s
+class=logging.Formatter

conf/logging_local.conf

@@ -0,0 +1,39 @@
+[loggers]
+keys=root, gunicorn.error, gunicorn.access, application.profiler
+
+[handlers]
+keys=console
+
+[formatters]
+keys=generic
+
+[logger_application.profiler]
+level=DEBUG
+handlers=console
+propagate=0
+qualname=application.profiler
+
+[logger_root]
+level=DEBUG
+handlers=console
+
+[logger_gunicorn.error]
+level=INFO
+handlers=console
+propagate=1
+qualname=gunicorn.error
+
+[logger_gunicorn.access]
+level=INFO
+handlers=console
+propagate=0
+qualname=gunicorn.access
+
+[handler_console]
+class=StreamHandler
+formatter=generic
+args=(sys.stdout, )
+
+[formatter_generic]
+format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s
+class=logging.Formatter

config.py

@@ -1,5 +1,4 @@
 import logging
-import logstash_formatter
 import requests
 import os.path
 
@@ -16,18 +15,6 @@ def build_requests_session():
   return sess
 
 
-def logs_init_builder(level=logging.DEBUG,
-                      formatter=logstash_formatter.LogstashFormatter()):
-  @staticmethod
-  def init_logs():
-    handler = logging.StreamHandler()
-    root_logger = logging.getLogger('')
-    root_logger.setLevel(level)
-    handler.setFormatter(formatter)
-    root_logger.addHandler(handler)
-
-  return init_logs
-
 # The set of configuration key names that will be accessible in the client. Since these
 # values are set to the frontend, DO NOT PLACE ANY SECRETS OR KEYS in this list.
 CLIENT_WHITELIST = ['SERVER_HOSTNAME', 'PREFERRED_URL_SCHEME', 'GITHUB_CLIENT_ID',
@@ -53,7 +40,7 @@ class DefaultConfig(object):
   JSONIFY_PRETTYPRINT_REGULAR = False
   SESSION_COOKIE_SECURE = False
 
-  LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter())
+  LOGGING_LEVEL = 'DEBUG'
   SEND_FILE_MAX_AGE_DEFAULT = 0
   POPULATE_DB_TEST_DATA = True
   PREFERRED_URL_SCHEME = 'http'

initdb.py

@@ -489,7 +489,8 @@ def populate_database():
                              'service': trigger.service.name})
 
 if __name__ == '__main__':
-  app.config['LOGGING_CONFIG']()
+  log_level = getattr(logging, app.config['LOGGING_LEVEL'])
+  logging.basicConfig(level=log_level)
   initialize_database()
 
   if app.config.get('POPULATE_DB_TEST_DATA', False):

requirements-nover.txt

@@ -18,7 +18,6 @@ python-daemon
 paramiko
 python-digitalocean
 xhtml2pdf
-logstash_formatter
 redis
 hiredis
 git+https://github.com/DevTable/docker-py.git

requirements.txt

@@ -21,7 +21,7 @@ blinker==1.3
 boto==2.27.0
 git+https://github.com/DevTable/docker-py.git
 ecdsa==0.11
-gevent==1.0
+gevent==1.0.1
 greenlet==0.4.2
 gunicorn==18.0
 hiredis==0.1.3
@@ -29,11 +29,9 @@ html5lib==0.999
 itsdangerous==0.24
 jsonschema==2.3.0
 lockfile==0.9.1
-logstash-formatter==0.5.8
 loremipsum==1.0.2
 marisa-trie==0.6
 mixpanel-py==3.1.2
-mock==1.0.1
 git+https://github.com/NateFerrero/oauth2lib.git
 paramiko==1.13.0
 peewee==2.2.3

util/glogger.py

@@ -1,23 +0,0 @@
-import logging
-import logstash_formatter
-import gunicorn.glogging
-
-from gunicorn import util
-
-class LogstashLogger(gunicorn.glogging.Logger):
-  def _set_handler(self, log, output, fmt):
-    # remove previous gunicorn log handler
-    h = self._get_gunicorn_handler(log)
-    if h:
-      log.handlers.remove(h)
-
-    if output is not None:
-      if output == "-":
-        h = logging.StreamHandler()
-      else:
-        util.check_is_writeable(output)
-        h = logging.FileHandler(output)
-
-      h.setFormatter(logstash_formatter.LogstashFormatter())
-      h._gunicorn = True
-      log.addHandler(h)