First attempt at emitting logstash style logs.

README.md

@@ -19,8 +19,8 @@ sudo gdebi --n binary_dependencies/*.deb
 running:
 
 ```
-sudo mkdir -p /mnt/nginx/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/nginx.conf
-STACK=prod gunicorn -c gunicorn_config.py application:application
+sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/nginx.conf
+sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application
 ```
 
 start the workers:

application.py

@@ -1,10 +1,11 @@
 import logging
+import os
 
 from app import app as application
 
 
-logging.basicConfig(**application.config['LOGGING_CONFIG'])
-
+# Initialize logging
+application.config['LOGGING_CONFIG']()
 
 from endpoints.api import api
 from endpoints.index import index

config.py

@@ -1,5 +1,6 @@
 import logging
-import sys
+import os
+import logstash_formatter
 
 from peewee import MySQLDatabase, SqliteDatabase
 from storage.s3 import S3Storage
@@ -11,10 +12,6 @@ from test.teststorage import FakeStorage, FakeUserfiles
 from test import analytics as fake_analytics
 
 
-LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \
-             '%(funcName)s - %(message)s'
-
-
 class FlaskConfig(object):
   SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884'
 
@@ -138,12 +135,26 @@ class BuildNodeConfig(object):
   BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
 
 
+def logs_init_builder(level=logging.DEBUG, logfile=None):
+  @staticmethod
+  def init_logs():
+    if logfile:
+      handler = logging.FileHandler(logfile)
+    else:
+      handler = logging.StreamHandler()
+
+    root_logger = logging.getLogger('')
+    root_logger.setLevel(level)
+    formatter = logstash_formatter.LogstashFormatter()
+    handler.setFormatter(formatter)
+    root_logger.addHandler(handler)
+
+  return init_logs
+
+
 class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
                  FakeAnalytics, StripeTestConfig):
-  LOGGING_CONFIG = {
-    'level': logging.WARN,
-    'format': LOG_FORMAT
-  }
+  LOGGING_CONFIG = logs_init_builder(logging.WARN)
   POPULATE_DB_TEST_DATA = True
   TESTING = True
   INCLUDE_TEST_ENDPOINTS = True
@@ -152,10 +163,7 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
 class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
                   StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
                   DigitalOceanConfig, BuildNodeConfig, S3Userfiles):
-  LOGGING_CONFIG = {
-    'level': logging.DEBUG,
-    'format': LOG_FORMAT
-  }
+  LOGGING_CONFIG = logs_init_builder()
   SEND_FILE_MAX_AGE_DEFAULT = 0
   POPULATE_DB_TEST_DATA = True
   INCLUDE_TEST_ENDPOINTS = True
@@ -165,10 +173,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,
                         StripeLiveConfig, MixpanelTestConfig,
                         GitHubProdConfig, DigitalOceanConfig,
                         BuildNodeConfig, S3Userfiles):
-  LOGGING_CONFIG = {
-    'level': logging.DEBUG,
-    'format': LOG_FORMAT
-  }
+  LOGGING_CONFIG = logs_init_builder()
   SEND_FILE_MAX_AGE_DEFAULT = 0
 
 
@@ -176,10 +181,6 @@ class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL,
                        StripeLiveConfig, MixpanelProdConfig,
                        GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig,
                        S3Userfiles):
-  LOGGING_CONFIG = {
-    'stream': sys.stderr,
-    'level': logging.DEBUG,
-    'format': LOG_FORMAT,
-    'filename': 'application.log',
-  }
+
+  LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log')
   SEND_FILE_MAX_AGE_DEFAULT = 0

initdb.py

@@ -350,7 +350,7 @@ def populate_database():
                    metadata={'token_code': 'somecode', 'repo': 'orgrepo'})
 
 if __name__ == '__main__':
-  logging.basicConfig(**app.config['LOGGING_CONFIG'])
+  app.config['LOGGING_CONFIG']()
   initialize_database()
 
   if app.config.get('POPULATE_DB_TEST_DATA', False):

nginx-staging.conf

@@ -1,8 +1,8 @@
 worker_processes 2;
 
 user root nogroup;
-pid /mnt/nginx/nginx.pid;
-error_log /mnt/nginx/nginx.error.log;
+pid /mnt/logs/nginx.pid;
+error_log /mnt/logs/nginx.error.log;
 
 events {
     worker_connections 1024;
@@ -14,7 +14,7 @@ http {
     include /usr/local/nginx/conf/mime.types.default;
 
     default_type application/octet-stream;
-    access_log /mnt/nginx/nginx.access.log combined;
+    access_log /mnt/logs/nginx.access.log combined;
     sendfile on;
 
     root /root/quay/;
@@ -43,7 +43,7 @@ http {
     server {
         listen 443 default;
         client_max_body_size 8G;
-        client_body_temp_path /mnt/nginx/client_body 1 2;
+        client_body_temp_path /mnt/logs/client_body 1 2;
         server_name _;
 
         keepalive_timeout 5;

nginx.conf

@@ -1,8 +1,8 @@
 worker_processes 8;
 
 user nobody nogroup;
-pid /mnt/nginx/nginx.pid;
-error_log /mnt/nginx/nginx.error.log;
+pid /mnt/logs/nginx.pid;
+error_log /mnt/logs/nginx.error.log;
 
 events {
     worker_connections 1024;
@@ -14,7 +14,7 @@ http {
     include /usr/local/nginx/conf/mime.types.default;
 
     default_type application/octet-stream;
-    access_log /mnt/nginx/nginx.access.log combined;
+    access_log /mnt/logs/nginx.access.log combined;
     sendfile on;
 
     gzip on;
@@ -41,7 +41,7 @@ http {
     server {
         listen 443 default;
         client_max_body_size 8G;
-        client_body_temp_path /mnt/nginx/client_body 1 2;
+        client_body_temp_path /mnt/logs/client_body 1 2;
         server_name _;
 
         keepalive_timeout 5;

requirements-nover.txt

@@ -18,3 +18,4 @@ python-daemon
 paramiko
 python-digitalocean
 xhtml2pdf
+logstash_formatter

requirements.txt

@@ -1,9 +1,9 @@
-APScheduler==2.1.1
+APScheduler==2.1.2
 Flask==0.10.1
 Flask-Login==0.2.9
 Flask-Mail==0.9.0
 Flask-Principal==0.4.0
-Jinja2==2.7.1
+Jinja2==2.7.2
 MarkupSafe==0.18
 Pillow==2.3.0
 PyMySQL==0.6.1
@@ -11,19 +11,20 @@ Werkzeug==0.9.4
 argparse==1.2.1
 beautifulsoup4==4.3.2
 blinker==1.3
-boto==2.21.2
+boto==2.24.0
 distribute==0.6.34
 ecdsa==0.10
 gevent==1.0
-greenlet==0.4.1
+greenlet==0.4.2
 gunicorn==18.0
 html5lib==1.0b3
 itsdangerous==0.23
 lockfile==0.9.1
+logstash-formatter==0.5.8
 marisa-trie==0.5.1
-mixpanel-py==3.0.0
-paramiko==1.12.0
-peewee==2.1.7
+mixpanel-py==3.1.1
+paramiko==1.12.1
+peewee==2.2.0
 py-bcrypt==0.4
 pyPdf==1.13
 pycrypto==2.6.1
@@ -31,8 +32,8 @@ python-daemon==1.6
 python-dateutil==2.2
 python-digitalocean==0.6
 reportlab==2.7
-requests==2.1.0
-six==1.4.1
-stripe==1.11.0
+requests==2.2.1
+six==1.5.2
+stripe==1.12.0
 wsgiref==0.1.2
 xhtml2pdf==0.0.5