Merge remote-tracking branch 'origin/logcabin'

Conflicts:
	application.py
	requirements.txt

README.md

@@ -14,13 +14,22 @@ virtualenv --distribute venv
 source venv/bin/activate
 pip install -r requirements.txt
 sudo gdebi --n binary_dependencies/*.deb
+sudo cp conf/logrotate/* /etc/logrotate.d/
 ```
 
 running:
 
 ```
-sudo mkdir -p /mnt/nginx/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/nginx.conf
-STACK=prod gunicorn -c gunicorn_config.py application:application
+sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf
+sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application
+```
+
+start the log shipper:
+
+```
+curl -s https://get.docker.io/ubuntu/ | sudo sh
+sudo docker pull quay.io/quay/logstash
+sudo docker run -d -e REDIS_PORT_6379_TCP_ADDR=logs.quay.io -v /mnt/logs:/mnt/logs quay.io/quay/logstash quay.conf
 ```
 
 start the workers:
@@ -35,7 +44,7 @@ bouncing the servers:
 
 ```
 sudo kill -HUP <pid of nginx>
-kill -HUP <pid of gunicorn>
+kill -HUP `cat /mnt/logs/gunicorn.pid`
 
 kill <pids of worker daemons>
 restart daemons

application.py

@@ -1,15 +1,16 @@
 import logging
+import os
 
 from app import app as application
 from data.model import db as model_db
 
 
-logging.basicConfig(**application.config['LOGGING_CONFIG'])
+# Initialize logging
+application.config['LOGGING_CONFIG']()
 
 # Turn off debug logging for boto
 logging.getLogger('boto').setLevel(logging.CRITICAL)
 
-
 from endpoints.api import api
 from endpoints.index import index
 from endpoints.web import web

conf/hosted-http-base.conf

@@ -0,0 +1,5 @@
+server {
+    listen 80 default_server;
+    server_name _;
+    rewrite ^ https://$host$request_uri? permanent;
+}

conf/http-base.conf

@@ -0,0 +1,33 @@
+log_format logstash_json '{ "@timestamp": "$time_iso8601", '
+                         '"@fields": { '
+                         '"remote_addr": "$remote_addr", '
+                         '"remote_user": "$remote_user", '
+                         '"body_bytes_sent": "$body_bytes_sent", '
+                         '"request_time": "$request_time", '
+                         '"status": "$status", '
+                         '"request": "$request", '
+                         '"request_method": "$request_method", '
+                         '"http_referrer": "$http_referer", '
+                         '"http_user_agent": "$http_user_agent" } }';
+
+types_hash_max_size 2048;
+include /usr/local/nginx/conf/mime.types.default;
+
+default_type application/octet-stream;
+access_log /mnt/logs/nginx.access.log logstash_json;
+sendfile on;
+
+gzip on;
+gzip_http_version 1.0;
+gzip_proxied any;
+gzip_min_length 500;
+gzip_disable "MSIE [1-6]\.";
+gzip_types text/plain text/xml text/css
+           text/javascript application/x-javascript
+           application/octet-stream;
+
+upstream app_server {
+    server unix:/tmp/gunicorn.sock fail_timeout=0;
+    # For a TCP configuration:
+    # server 192.168.0.7:8000 fail_timeout=0;
+}

conf/logrotate/quay-logrotate

@@ -0,0 +1,33 @@
+/mnt/logs/nginx.access.log {
+  daily
+  rotate 7
+  compress
+  delaycompress
+  missingok
+  notifempty
+  create 644 root root
+}
+
+/mnt/logs/nginx.error.log {
+  daily
+  rotate 7
+  compress
+  delaycompress
+  missingok
+  notifempty
+  create 644 root root
+}
+
+/mnt/logs/application.log {
+  daily
+  rotate 7
+  compress
+  delaycompress
+  missingok
+  notifempty
+  create 644 root root
+
+  postrotate
+    kill -USR1 `cat /mnt/logs/gunicorn.pid`
+  endscript
+}

conf/nginx-local.conf

@@ -0,0 +1,18 @@
+include root-base.conf;
+
+worker_processes 2;
+
+http {
+    include http-base.conf;
+
+    server {
+        include server-base.conf;
+
+        listen 5000 default;
+
+        location /static/ {
+            # checks for static file, if not found proxy to app
+            alias /home/jake/Projects/docker/quay/static/;
+        }
+    }
+}

conf/nginx-staging.conf

@@ -0,0 +1,30 @@
+include root-base.conf;
+
+worker_processes 2;
+
+user root nogroup;
+
+http {
+    include http-base.conf;
+
+    include hosted-http-base.conf;
+
+    server {
+        include server-base.conf;
+
+        listen 443 default;
+
+        ssl on;
+        ssl_certificate ./certs/quay-staging-unified.cert;
+        ssl_certificate_key ./certs/quay-staging.key;
+        ssl_session_timeout 5m;
+        ssl_protocols SSLv3 TLSv1;
+        ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP;
+        ssl_prefer_server_ciphers on;
+
+        location /static/ {
+            # checks for static file, if not found proxy to app
+            alias /root/quay/static/;
+        }
+    }
+}

conf/nginx.conf

@@ -0,0 +1,30 @@
+include root-base.conf;
+
+worker_processes 8;
+
+user nobody nogroup;
+
+http {
+    include http-base.conf;
+
+    include hosted-http-base.conf;
+
+    server {
+        include server-base.conf;
+
+        listen 443 default;
+
+        ssl on;
+        ssl_certificate ./certs/quay-unified.cert;
+        ssl_certificate_key ./certs/quay.key;
+        ssl_session_timeout 5m;
+        ssl_protocols SSLv3 TLSv1;
+        ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP;
+        ssl_prefer_server_ciphers on;
+
+        location /static/ {
+            # checks for static file, if not found proxy to app
+            alias /home/ubuntu/quay/static/;
+        }
+    }
+}

conf/root-base.conf

@@ -0,0 +1,7 @@
+pid /mnt/logs/nginx.pid;
+error_log /mnt/logs/nginx.error.log;
+
+events {
+    worker_connections 1024;
+    accept_mutex off;
+}

conf/server-base.conf

@@ -0,0 +1,24 @@
+client_max_body_size 8G;
+client_body_temp_path /mnt/logs/client_body 1 2;
+server_name _;
+
+keepalive_timeout 5;
+
+if ($args ~ "_escaped_fragment_") {
+    rewrite ^ /snapshot$uri;
+}
+
+location / {
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Host $http_host;
+    proxy_redirect off;
+    proxy_buffering off;
+
+    proxy_request_buffering off;
+    proxy_set_header Transfer-Encoding $http_transfer_encoding;
+
+    proxy_pass   http://app_server;
+    proxy_read_timeout 2000;
+    proxy_temp_path /mnt/logs/proxy_temp 1 2;
+}

config.py

@@ -1,5 +1,6 @@
 import logging
-import sys
+import os
+import logstash_formatter
 
 from peewee import MySQLDatabase, SqliteDatabase
 from storage.s3 import S3Storage
@@ -11,10 +12,6 @@ from test.teststorage import FakeStorage, FakeUserfiles
 from test import analytics as fake_analytics
 
 
-LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \
-             '%(funcName)s - %(message)s'
-
-
 class FlaskConfig(object):
   SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884'
 
@@ -138,12 +135,26 @@ class BuildNodeConfig(object):
   BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
 
 
+def logs_init_builder(level=logging.DEBUG, logfile=None):
+  @staticmethod
+  def init_logs():
+    if logfile:
+      handler = logging.FileHandler(logfile)
+    else:
+      handler = logging.StreamHandler()
+
+    root_logger = logging.getLogger('')
+    root_logger.setLevel(level)
+    formatter = logstash_formatter.LogstashFormatter()
+    handler.setFormatter(formatter)
+    root_logger.addHandler(handler)
+
+  return init_logs
+
+
 class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
                  FakeAnalytics, StripeTestConfig):
-  LOGGING_CONFIG = {
-    'level': logging.WARN,
-    'format': LOG_FORMAT
-  }
+  LOGGING_CONFIG = logs_init_builder(logging.WARN)
   POPULATE_DB_TEST_DATA = True
   TESTING = True
   INCLUDE_TEST_ENDPOINTS = True
@@ -152,10 +163,7 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
 class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
                   StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
                   DigitalOceanConfig, BuildNodeConfig, S3Userfiles):
-  LOGGING_CONFIG = {
-    'level': logging.DEBUG,
-    'format': LOG_FORMAT
-  }
+  LOGGING_CONFIG = logs_init_builder()
   SEND_FILE_MAX_AGE_DEFAULT = 0
   POPULATE_DB_TEST_DATA = True
   INCLUDE_TEST_ENDPOINTS = True
@@ -165,10 +173,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,
                         StripeLiveConfig, MixpanelTestConfig,
                         GitHubProdConfig, DigitalOceanConfig,
                         BuildNodeConfig, S3Userfiles):
-  LOGGING_CONFIG = {
-    'level': logging.DEBUG,
-    'format': LOG_FORMAT
-  }
+  LOGGING_CONFIG = logs_init_builder()
   SEND_FILE_MAX_AGE_DEFAULT = 0
 
 
@@ -176,10 +181,6 @@ class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL,
                        StripeLiveConfig, MixpanelProdConfig,
                        GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig,
                        S3Userfiles):
-  LOGGING_CONFIG = {
-    'stream': sys.stderr,
-    'level': logging.DEBUG,
-    'format': LOG_FORMAT,
-    'filename': 'application.log',
-  }
+
+  LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log')
   SEND_FILE_MAX_AGE_DEFAULT = 0

gunicorn_config.py

@@ -3,3 +3,4 @@ workers = 8
 worker_class = 'gevent'
 timeout = 2000
 daemon = True
+pidfile = '/mnt/logs/gunicorn.pid'

initdb.py

@@ -388,7 +388,7 @@ def populate_database():
                    metadata={'token_code': 'somecode', 'repo': 'orgrepo'})
 
 if __name__ == '__main__':
-  logging.basicConfig(**app.config['LOGGING_CONFIG'])
+  app.config['LOGGING_CONFIG']()
   initialize_database()
 
   if app.config.get('POPULATE_DB_TEST_DATA', False):

nginx-staging.conf

@@ -1,83 +0,0 @@
-worker_processes 2;
-
-user root nogroup;
-pid /mnt/nginx/nginx.pid;
-error_log /mnt/nginx/nginx.error.log;
-
-events {
-    worker_connections 1024;
-    accept_mutex off;
-}
-
-http {
-    types_hash_max_size 2048;
-    include /usr/local/nginx/conf/mime.types.default;
-
-    default_type application/octet-stream;
-    access_log /mnt/nginx/nginx.access.log combined;
-    sendfile on;
-
-    root /root/quay/;
-    
-    gzip on;
-    gzip_http_version 1.0;
-    gzip_proxied any;
-    gzip_min_length 500;
-    gzip_disable "MSIE [1-6]\.";
-    gzip_types text/plain text/xml text/css
-               text/javascript application/x-javascript
-               application/octet-stream;
-
-    upstream app_server {
-        server unix:/tmp/gunicorn.sock fail_timeout=0;
-        # For a TCP configuration:
-        # server 192.168.0.7:8000 fail_timeout=0;
-    }
-
-    server {
-        listen 80 default_server;
-        server_name _;
-        rewrite ^ https://$host$request_uri? permanent;
-    }
-
-    server {
-        listen 443 default;
-        client_max_body_size 8G;
-        client_body_temp_path /mnt/nginx/client_body 1 2;
-        server_name _;
-
-        keepalive_timeout 5;
-
-        ssl on;
-        ssl_certificate ./certs/quay-staging-unified.cert;
-        ssl_certificate_key ./certs/quay-staging.key;
-        ssl_session_timeout 5m;
-        ssl_protocols SSLv3 TLSv1;
-        ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP;
-        ssl_prefer_server_ciphers on;
-
-        if ($args ~ "_escaped_fragment_") {
-            rewrite ^ /snapshot$uri;
-        }
-
-        location /static/ {
-            # checks for static file, if not found proxy to app
-            alias /root/quay/static/;
-        }
-
-        location / {
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            proxy_set_header Host $http_host;
-            proxy_redirect off;
-            proxy_buffering off;
-
-            proxy_request_buffering off;
-            proxy_set_header Transfer-Encoding $http_transfer_encoding;
-
-            proxy_pass   http://app_server;
-            proxy_read_timeout 2000;
-            proxy_temp_path /mnt/nginx/proxy_temp 1 2;
-        }
-    }
-}

nginx.conf

@@ -1,81 +0,0 @@
-worker_processes 8;
-
-user nobody nogroup;
-pid /mnt/nginx/nginx.pid;
-error_log /mnt/nginx/nginx.error.log;
-
-events {
-    worker_connections 1024;
-    accept_mutex off;
-}
-
-http {
-    types_hash_max_size 2048;
-    include /usr/local/nginx/conf/mime.types.default;
-
-    default_type application/octet-stream;
-    access_log /mnt/nginx/nginx.access.log combined;
-    sendfile on;
-
-    gzip on;
-    gzip_http_version 1.0;
-    gzip_proxied any;
-    gzip_min_length 500;
-    gzip_disable "MSIE [1-6]\.";
-    gzip_types text/plain text/xml text/css
-               text/javascript application/x-javascript
-               application/octet-stream;
-
-    upstream app_server {
-        server unix:/tmp/gunicorn.sock fail_timeout=0;
-        # For a TCP configuration:
-        # server 192.168.0.7:8000 fail_timeout=0;
-    }
-
-    server {
-        listen 80 default_server;
-        server_name _;
-        rewrite ^ https://$host$request_uri? permanent;
-    }
-
-    server {
-        listen 443 default;
-        client_max_body_size 8G;
-        client_body_temp_path /mnt/nginx/client_body 1 2;
-        server_name _;
-
-        keepalive_timeout 5;
-
-        ssl on;
-        ssl_certificate ./certs/quay-unified.cert;
-        ssl_certificate_key ./certs/quay.key;
-        ssl_session_timeout 5m;
-        ssl_protocols SSLv3 TLSv1;
-        ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP;
-        ssl_prefer_server_ciphers on;
-
-        if ($args ~ "_escaped_fragment_") {
-            rewrite ^ /snapshot$uri;
-        }
-
-        location /static/ {
-            # checks for static file, if not found proxy to app
-            alias /home/ubuntu/quay/static/;
-        }
-
-        location / {
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-            proxy_set_header Host $http_host;
-            proxy_redirect off;
-            proxy_buffering off;
-
-            proxy_request_buffering off;
-            proxy_set_header Transfer-Encoding $http_transfer_encoding;
-
-            proxy_pass   http://app_server;
-            proxy_read_timeout 2000;
-            proxy_temp_path /mnt/nginx/proxy_temp 1 2;
-        }
-    }
-}

requirements-nover.txt

@@ -17,4 +17,5 @@ apscheduler
 python-daemon
 paramiko
 python-digitalocean
-xhtml2pdf
+xhtml2pdf
+logstash_formatter

requirements.txt

@@ -1,9 +1,9 @@
-APScheduler==2.1.1
+APScheduler==2.1.2
 Flask==0.10.1
 Flask-Login==0.2.9
 Flask-Mail==0.9.0
 Flask-Principal==0.4.0
-Jinja2==2.7.1
+Jinja2==2.7.2
 MarkupSafe==0.18
 Pillow==2.3.0
 PyMySQL==0.6.1
@@ -11,18 +11,19 @@ Werkzeug==0.9.4
 argparse==1.2.1
 beautifulsoup4==4.3.2
 blinker==1.3
-boto==2.21.2
+boto==2.24.0
 distribute==0.6.34
 ecdsa==0.10
 gevent==1.0
-greenlet==0.4.1
+greenlet==0.4.2
 gunicorn==18.0
 html5lib==1.0b3
 itsdangerous==0.23
 lockfile==0.9.1
+logstash-formatter==0.5.8
 marisa-trie==0.5.1
-mixpanel-py==3.0.0
-paramiko==1.12.0
+mixpanel-py==3.1.1
+paramiko==1.12.1
 peewee==2.2.0
 py-bcrypt==0.4
 pyPdf==1.13
@@ -31,8 +32,8 @@ python-daemon==1.6
 python-dateutil==2.2
 python-digitalocean==0.6
 reportlab==2.7
-requests==2.1.0
-six==1.4.1
-stripe==1.11.0
+requests==2.2.1
+six==1.5.2
+stripe==1.12.0
 wsgiref==0.1.2
 xhtml2pdf==0.0.5