From a5a61576ae7c2d49ea93ae291b6152646c3149e3 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 1 May 2014 19:44:28 -0400 Subject: [PATCH] Revamp the logging a bit. Not quite done yet. --- application.py | 11 +++++++---- conf/gunicorn_config.py | 4 +--- conf/gunicorn_local.py | 4 +--- conf/logging.conf | 39 +++++++++++++++++++++++++++++++++++++++ conf/logging_local.conf | 39 +++++++++++++++++++++++++++++++++++++++ config.py | 15 +-------------- initdb.py | 3 ++- requirements-nover.txt | 1 - requirements.txt | 4 +--- util/glogger.py | 23 ----------------------- 10 files changed, 91 insertions(+), 52 deletions(-) create mode 100644 conf/logging.conf create mode 100644 conf/logging_local.conf delete mode 100644 util/glogger.py diff --git a/application.py b/application.py index afc503379..a5bb45edf 100644 --- a/application.py +++ b/application.py @@ -1,10 +1,8 @@ import logging +import logging.config from app import app as application -# Initialize logging -application.config['LOGGING_CONFIG']() - from data.model import db as model_db # Turn off debug logging for boto @@ -41,4 +39,9 @@ def close_db(exc): application.teardown_request(close_db) if __name__ == '__main__': - application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') + logging.config.fileConfig('conf/logging_local.conf', disable_existing_loggers=False) + + profile = logging.getLogger('application.profiler') + profile.debug('This is a profiling statement') + + application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') \ No newline at end of file diff --git a/conf/gunicorn_config.py b/conf/gunicorn_config.py index a74f95786..b86250125 100644 --- a/conf/gunicorn_config.py +++ b/conf/gunicorn_config.py @@ -3,7 +3,5 @@ workers = 8 worker_class = 'gevent' timeout = 2000 pidfile = '/tmp/gunicorn.pid' -errorlog = '/mnt/logs/application.log' -loglevel = 'debug' -logger_class = 'util.glogger.LogstashLogger' +logconfig = 'conf/logging.conf' pythonpath = '.' \ No newline at end of file diff --git a/conf/gunicorn_local.py b/conf/gunicorn_local.py index 2a145fd98..9f93eb008 100644 --- a/conf/gunicorn_local.py +++ b/conf/gunicorn_local.py @@ -3,7 +3,5 @@ workers = 2 worker_class = 'gevent' timeout = 2000 daemon = False -errorlog = '-' -loglevel = 'debug' -logger_class = 'util.glogger.LogstashLogger' +logconfig = 'conf/logging_local.conf' pythonpath = '.' \ No newline at end of file diff --git a/conf/logging.conf b/conf/logging.conf new file mode 100644 index 000000000..fa23cf86a --- /dev/null +++ b/conf/logging.conf @@ -0,0 +1,39 @@ +[loggers] +keys=root, gunicorn.error, gunicorn.access + +[handlers] +keys=error_file + +[formatters] +keys=generic + +[logger_application.profiler] +level=DEBUG +handlers=error_file +propagate=0 +qualname=application.profiler + +[logger_root] +level=DEBUG +handlers=error_file + +[logger_gunicorn.error] +level=INFO +handlers=error_file +propagate=1 +qualname=gunicorn.error + +[logger_gunicorn.access] +level=INFO +handlers=error_file +propagate=0 +qualname=gunicorn.access + +[handler_error_file] +class=logging.FileHandler +formatter=generic +args=('/mnt/logs/application.log',) + +[formatter_generic] +format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s +class=logging.Formatter diff --git a/conf/logging_local.conf b/conf/logging_local.conf new file mode 100644 index 000000000..e56efee24 --- /dev/null +++ b/conf/logging_local.conf @@ -0,0 +1,39 @@ +[loggers] +keys=root, gunicorn.error, gunicorn.access, application.profiler + +[handlers] +keys=console + +[formatters] +keys=generic + +[logger_application.profiler] +level=DEBUG +handlers=console +propagate=0 +qualname=application.profiler + +[logger_root] +level=DEBUG +handlers=console + +[logger_gunicorn.error] +level=INFO +handlers=console +propagate=1 +qualname=gunicorn.error + +[logger_gunicorn.access] +level=INFO +handlers=console +propagate=0 +qualname=gunicorn.access + +[handler_console] +class=StreamHandler +formatter=generic +args=(sys.stdout, ) + +[formatter_generic] +format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s +class=logging.Formatter diff --git a/config.py b/config.py index f08e555db..74181fdae 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,4 @@ import logging -import logstash_formatter import requests import os.path @@ -16,18 +15,6 @@ def build_requests_session(): return sess -def logs_init_builder(level=logging.DEBUG, - formatter=logstash_formatter.LogstashFormatter()): - @staticmethod - def init_logs(): - handler = logging.StreamHandler() - root_logger = logging.getLogger('') - root_logger.setLevel(level) - handler.setFormatter(formatter) - root_logger.addHandler(handler) - - return init_logs - # The set of configuration key names that will be accessible in the client. Since these # values are set to the frontend, DO NOT PLACE ANY SECRETS OR KEYS in this list. CLIENT_WHITELIST = ['SERVER_HOSTNAME', 'PREFERRED_URL_SCHEME', 'GITHUB_CLIENT_ID', @@ -53,7 +40,7 @@ class DefaultConfig(object): JSONIFY_PRETTYPRINT_REGULAR = False SESSION_COOKIE_SECURE = False - LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter()) + LOGGING_LEVEL = 'DEBUG' SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True PREFERRED_URL_SCHEME = 'http' diff --git a/initdb.py b/initdb.py index cf330aabe..2570b7ca9 100644 --- a/initdb.py +++ b/initdb.py @@ -489,7 +489,8 @@ def populate_database(): 'service': trigger.service.name}) if __name__ == '__main__': - app.config['LOGGING_CONFIG']() + log_level = getattr(logging, app.config['LOGGING_LEVEL']) + logging.basicConfig(level=log_level) initialize_database() if app.config.get('POPULATE_DB_TEST_DATA', False): diff --git a/requirements-nover.txt b/requirements-nover.txt index f038ba905..cc370da9d 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -18,7 +18,6 @@ python-daemon paramiko python-digitalocean xhtml2pdf -logstash_formatter redis hiredis git+https://github.com/DevTable/docker-py.git diff --git a/requirements.txt b/requirements.txt index daea582a9..7951f5dd8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ blinker==1.3 boto==2.27.0 git+https://github.com/DevTable/docker-py.git ecdsa==0.11 -gevent==1.0 +gevent==1.0.1 greenlet==0.4.2 gunicorn==18.0 hiredis==0.1.3 @@ -29,11 +29,9 @@ html5lib==0.999 itsdangerous==0.24 jsonschema==2.3.0 lockfile==0.9.1 -logstash-formatter==0.5.8 loremipsum==1.0.2 marisa-trie==0.6 mixpanel-py==3.1.2 -mock==1.0.1 git+https://github.com/NateFerrero/oauth2lib.git paramiko==1.13.0 peewee==2.2.3 diff --git a/util/glogger.py b/util/glogger.py deleted file mode 100644 index 44303bfdb..000000000 --- a/util/glogger.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging -import logstash_formatter -import gunicorn.glogging - -from gunicorn import util - -class LogstashLogger(gunicorn.glogging.Logger): - def _set_handler(self, log, output, fmt): - # remove previous gunicorn log handler - h = self._get_gunicorn_handler(log) - if h: - log.handlers.remove(h) - - if output is not None: - if output == "-": - h = logging.StreamHandler() - else: - util.check_is_writeable(output) - h = logging.FileHandler(output) - - h.setFormatter(logstash_formatter.LogstashFormatter()) - h._gunicorn = True - log.addHandler(h)