From c13654bb52f0857562f69d1a710244d606d8c0a3 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Thu, 30 Jan 2014 13:32:06 -0500 Subject: [PATCH] First attempt at emitting logstash style logs. --- README.md | 4 ++-- application.py | 5 +++-- config.py | 47 +++++++++++++++++++++--------------------- initdb.py | 2 +- nginx-staging.conf | 8 +++---- nginx.conf | 8 +++---- requirements-nover.txt | 3 ++- requirements.txt | 21 ++++++++++--------- 8 files changed, 51 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 7018c76a1..fab774082 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ sudo gdebi --n binary_dependencies/*.deb running: ``` -sudo mkdir -p /mnt/nginx/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/nginx.conf -STACK=prod gunicorn -c gunicorn_config.py application:application +sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/nginx.conf +sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application ``` start the workers: diff --git a/application.py b/application.py index 26101f50b..e36c445f1 100644 --- a/application.py +++ b/application.py @@ -1,10 +1,11 @@ import logging +import os from app import app as application -logging.basicConfig(**application.config['LOGGING_CONFIG']) - +# Initialize logging +application.config['LOGGING_CONFIG']() from endpoints.api import api from endpoints.index import index diff --git a/config.py b/config.py index 90705d337..55492bab1 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ import logging -import sys +import os +import logstash_formatter from peewee import MySQLDatabase, SqliteDatabase from storage.s3 import S3Storage @@ -11,10 +12,6 @@ from test.teststorage import FakeStorage, FakeUserfiles from test import analytics as fake_analytics -LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \ - '%(funcName)s - %(message)s' - - class FlaskConfig(object): SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884' @@ -138,12 +135,26 @@ class BuildNodeConfig(object): BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G' +def logs_init_builder(level=logging.DEBUG, logfile=None): + @staticmethod + def init_logs(): + if logfile: + handler = logging.FileHandler(logfile) + else: + handler = logging.StreamHandler() + + root_logger = logging.getLogger('') + root_logger.setLevel(level) + formatter = logstash_formatter.LogstashFormatter() + handler.setFormatter(formatter) + root_logger.addHandler(handler) + + return init_logs + + class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, FakeAnalytics, StripeTestConfig): - LOGGING_CONFIG = { - 'level': logging.WARN, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder(logging.WARN) POPULATE_DB_TEST_DATA = True TESTING = True INCLUDE_TEST_ENDPOINTS = True @@ -152,10 +163,7 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'level': logging.DEBUG, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True INCLUDE_TEST_ENDPOINTS = True @@ -165,10 +173,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'level': logging.DEBUG, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 @@ -176,10 +181,6 @@ class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'stream': sys.stderr, - 'level': logging.DEBUG, - 'format': LOG_FORMAT, - 'filename': 'application.log', - } + + LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log') SEND_FILE_MAX_AGE_DEFAULT = 0 diff --git a/initdb.py b/initdb.py index a3ee26362..430f041f4 100644 --- a/initdb.py +++ b/initdb.py @@ -350,7 +350,7 @@ def populate_database(): metadata={'token_code': 'somecode', 'repo': 'orgrepo'}) if __name__ == '__main__': - logging.basicConfig(**app.config['LOGGING_CONFIG']) + app.config['LOGGING_CONFIG']() initialize_database() if app.config.get('POPULATE_DB_TEST_DATA', False): diff --git a/nginx-staging.conf b/nginx-staging.conf index b29bda2c5..57c9bb53f 100644 --- a/nginx-staging.conf +++ b/nginx-staging.conf @@ -1,8 +1,8 @@ worker_processes 2; user root nogroup; -pid /mnt/nginx/nginx.pid; -error_log /mnt/nginx/nginx.error.log; +pid /mnt/logs/nginx.pid; +error_log /mnt/logs/nginx.error.log; events { worker_connections 1024; @@ -14,7 +14,7 @@ http { include /usr/local/nginx/conf/mime.types.default; default_type application/octet-stream; - access_log /mnt/nginx/nginx.access.log combined; + access_log /mnt/logs/nginx.access.log combined; sendfile on; root /root/quay/; @@ -43,7 +43,7 @@ http { server { listen 443 default; client_max_body_size 8G; - client_body_temp_path /mnt/nginx/client_body 1 2; + client_body_temp_path /mnt/logs/client_body 1 2; server_name _; keepalive_timeout 5; diff --git a/nginx.conf b/nginx.conf index 53cd3c9f8..e82675461 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,8 +1,8 @@ worker_processes 8; user nobody nogroup; -pid /mnt/nginx/nginx.pid; -error_log /mnt/nginx/nginx.error.log; +pid /mnt/logs/nginx.pid; +error_log /mnt/logs/nginx.error.log; events { worker_connections 1024; @@ -14,7 +14,7 @@ http { include /usr/local/nginx/conf/mime.types.default; default_type application/octet-stream; - access_log /mnt/nginx/nginx.access.log combined; + access_log /mnt/logs/nginx.access.log combined; sendfile on; gzip on; @@ -41,7 +41,7 @@ http { server { listen 443 default; client_max_body_size 8G; - client_body_temp_path /mnt/nginx/client_body 1 2; + client_body_temp_path /mnt/logs/client_body 1 2; server_name _; keepalive_timeout 5; diff --git a/requirements-nover.txt b/requirements-nover.txt index f42a8a90d..c430edf5a 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -17,4 +17,5 @@ apscheduler python-daemon paramiko python-digitalocean -xhtml2pdf \ No newline at end of file +xhtml2pdf +logstash_formatter diff --git a/requirements.txt b/requirements.txt index ce9a22c54..7438e6dce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -APScheduler==2.1.1 +APScheduler==2.1.2 Flask==0.10.1 Flask-Login==0.2.9 Flask-Mail==0.9.0 Flask-Principal==0.4.0 -Jinja2==2.7.1 +Jinja2==2.7.2 MarkupSafe==0.18 Pillow==2.3.0 PyMySQL==0.6.1 @@ -11,19 +11,20 @@ Werkzeug==0.9.4 argparse==1.2.1 beautifulsoup4==4.3.2 blinker==1.3 -boto==2.21.2 +boto==2.24.0 distribute==0.6.34 ecdsa==0.10 gevent==1.0 -greenlet==0.4.1 +greenlet==0.4.2 gunicorn==18.0 html5lib==1.0b3 itsdangerous==0.23 lockfile==0.9.1 +logstash-formatter==0.5.8 marisa-trie==0.5.1 -mixpanel-py==3.0.0 -paramiko==1.12.0 -peewee==2.1.7 +mixpanel-py==3.1.1 +paramiko==1.12.1 +peewee==2.2.0 py-bcrypt==0.4 pyPdf==1.13 pycrypto==2.6.1 @@ -31,8 +32,8 @@ python-daemon==1.6 python-dateutil==2.2 python-digitalocean==0.6 reportlab==2.7 -requests==2.1.0 -six==1.4.1 -stripe==1.11.0 +requests==2.2.1 +six==1.5.2 +stripe==1.12.0 wsgiref==0.1.2 xhtml2pdf==0.0.5