diff --git a/README.md b/README.md index 7018c76a1..e082d4dda 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,22 @@ virtualenv --distribute venv source venv/bin/activate pip install -r requirements.txt sudo gdebi --n binary_dependencies/*.deb +sudo cp conf/logrotate/* /etc/logrotate.d/ ``` running: ``` -sudo mkdir -p /mnt/nginx/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/nginx.conf -STACK=prod gunicorn -c gunicorn_config.py application:application +sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf +sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application +``` + +start the log shipper: + +``` +curl -s https://get.docker.io/ubuntu/ | sudo sh +sudo docker pull quay.io/quay/logstash +sudo docker run -d -e REDIS_PORT_6379_TCP_ADDR=logs.quay.io -v /mnt/logs:/mnt/logs quay.io/quay/logstash quay.conf ``` start the workers: @@ -35,7 +44,7 @@ bouncing the servers: ``` sudo kill -HUP -kill -HUP +kill -HUP `cat /mnt/logs/gunicorn.pid` kill restart daemons diff --git a/application.py b/application.py index 7e6139ada..80411d50f 100644 --- a/application.py +++ b/application.py @@ -1,15 +1,16 @@ import logging +import os from app import app as application from data.model import db as model_db -logging.basicConfig(**application.config['LOGGING_CONFIG']) +# Initialize logging +application.config['LOGGING_CONFIG']() # Turn off debug logging for boto logging.getLogger('boto').setLevel(logging.CRITICAL) - from endpoints.api import api from endpoints.index import index from endpoints.web import web diff --git a/certs/digital_ocean b/conf/certs/digital_ocean similarity index 100% rename from certs/digital_ocean rename to conf/certs/digital_ocean diff --git a/certs/digital_ocean.pub b/conf/certs/digital_ocean.pub similarity index 100% rename from certs/digital_ocean.pub rename to conf/certs/digital_ocean.pub diff --git a/certs/quay-enc.key b/conf/certs/quay-enc.key similarity index 100% rename from certs/quay-enc.key rename to conf/certs/quay-enc.key diff --git a/certs/quay-staging-enc.key b/conf/certs/quay-staging-enc.key similarity index 100% rename from certs/quay-staging-enc.key rename to conf/certs/quay-staging-enc.key diff --git a/certs/quay-staging-unified.cert b/conf/certs/quay-staging-unified.cert similarity index 100% rename from certs/quay-staging-unified.cert rename to conf/certs/quay-staging-unified.cert diff --git a/certs/quay-staging.cert b/conf/certs/quay-staging.cert similarity index 100% rename from certs/quay-staging.cert rename to conf/certs/quay-staging.cert diff --git a/certs/quay-staging.key b/conf/certs/quay-staging.key similarity index 100% rename from certs/quay-staging.key rename to conf/certs/quay-staging.key diff --git a/certs/quay-unified.cert b/conf/certs/quay-unified.cert similarity index 100% rename from certs/quay-unified.cert rename to conf/certs/quay-unified.cert diff --git a/certs/quay.cert b/conf/certs/quay.cert similarity index 100% rename from certs/quay.cert rename to conf/certs/quay.cert diff --git a/certs/quay.key b/conf/certs/quay.key similarity index 100% rename from certs/quay.key rename to conf/certs/quay.key diff --git a/conf/hosted-http-base.conf b/conf/hosted-http-base.conf new file mode 100644 index 000000000..c3e910e8f --- /dev/null +++ b/conf/hosted-http-base.conf @@ -0,0 +1,5 @@ +server { + listen 80 default_server; + server_name _; + rewrite ^ https://$host$request_uri? permanent; +} diff --git a/conf/http-base.conf b/conf/http-base.conf new file mode 100644 index 000000000..32e8b3730 --- /dev/null +++ b/conf/http-base.conf @@ -0,0 +1,33 @@ +log_format logstash_json '{ "@timestamp": "$time_iso8601", ' + '"@fields": { ' + '"remote_addr": "$remote_addr", ' + '"remote_user": "$remote_user", ' + '"body_bytes_sent": "$body_bytes_sent", ' + '"request_time": "$request_time", ' + '"status": "$status", ' + '"request": "$request", ' + '"request_method": "$request_method", ' + '"http_referrer": "$http_referer", ' + '"http_user_agent": "$http_user_agent" } }'; + +types_hash_max_size 2048; +include /usr/local/nginx/conf/mime.types.default; + +default_type application/octet-stream; +access_log /mnt/logs/nginx.access.log logstash_json; +sendfile on; + +gzip on; +gzip_http_version 1.0; +gzip_proxied any; +gzip_min_length 500; +gzip_disable "MSIE [1-6]\."; +gzip_types text/plain text/xml text/css + text/javascript application/x-javascript + application/octet-stream; + +upstream app_server { + server unix:/tmp/gunicorn.sock fail_timeout=0; + # For a TCP configuration: + # server 192.168.0.7:8000 fail_timeout=0; +} diff --git a/conf/logrotate/quay-logrotate b/conf/logrotate/quay-logrotate new file mode 100644 index 000000000..313cdea13 --- /dev/null +++ b/conf/logrotate/quay-logrotate @@ -0,0 +1,33 @@ +/mnt/logs/nginx.access.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 644 root root +} + +/mnt/logs/nginx.error.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 644 root root +} + +/mnt/logs/application.log { + daily + rotate 7 + compress + delaycompress + missingok + notifempty + create 644 root root + + postrotate + kill -USR1 `cat /mnt/logs/gunicorn.pid` + endscript +} \ No newline at end of file diff --git a/conf/nginx-local.conf b/conf/nginx-local.conf new file mode 100644 index 000000000..0545399a0 --- /dev/null +++ b/conf/nginx-local.conf @@ -0,0 +1,18 @@ +include root-base.conf; + +worker_processes 2; + +http { + include http-base.conf; + + server { + include server-base.conf; + + listen 5000 default; + + location /static/ { + # checks for static file, if not found proxy to app + alias /home/jake/Projects/docker/quay/static/; + } + } +} diff --git a/conf/nginx-staging.conf b/conf/nginx-staging.conf new file mode 100644 index 000000000..f8fb03784 --- /dev/null +++ b/conf/nginx-staging.conf @@ -0,0 +1,30 @@ +include root-base.conf; + +worker_processes 2; + +user root nogroup; + +http { + include http-base.conf; + + include hosted-http-base.conf; + + server { + include server-base.conf; + + listen 443 default; + + ssl on; + ssl_certificate ./certs/quay-staging-unified.cert; + ssl_certificate_key ./certs/quay-staging.key; + ssl_session_timeout 5m; + ssl_protocols SSLv3 TLSv1; + ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; + ssl_prefer_server_ciphers on; + + location /static/ { + # checks for static file, if not found proxy to app + alias /root/quay/static/; + } + } +} diff --git a/conf/nginx.conf b/conf/nginx.conf new file mode 100644 index 000000000..896b151de --- /dev/null +++ b/conf/nginx.conf @@ -0,0 +1,30 @@ +include root-base.conf; + +worker_processes 8; + +user nobody nogroup; + +http { + include http-base.conf; + + include hosted-http-base.conf; + + server { + include server-base.conf; + + listen 443 default; + + ssl on; + ssl_certificate ./certs/quay-unified.cert; + ssl_certificate_key ./certs/quay.key; + ssl_session_timeout 5m; + ssl_protocols SSLv3 TLSv1; + ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; + ssl_prefer_server_ciphers on; + + location /static/ { + # checks for static file, if not found proxy to app + alias /home/ubuntu/quay/static/; + } + } +} diff --git a/conf/root-base.conf b/conf/root-base.conf new file mode 100644 index 000000000..16a63fda0 --- /dev/null +++ b/conf/root-base.conf @@ -0,0 +1,7 @@ +pid /mnt/logs/nginx.pid; +error_log /mnt/logs/nginx.error.log; + +events { + worker_connections 1024; + accept_mutex off; +} \ No newline at end of file diff --git a/conf/server-base.conf b/conf/server-base.conf new file mode 100644 index 000000000..f37d83bba --- /dev/null +++ b/conf/server-base.conf @@ -0,0 +1,24 @@ +client_max_body_size 8G; +client_body_temp_path /mnt/logs/client_body 1 2; +server_name _; + +keepalive_timeout 5; + +if ($args ~ "_escaped_fragment_") { + rewrite ^ /snapshot$uri; +} + +location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_buffering off; + + proxy_request_buffering off; + proxy_set_header Transfer-Encoding $http_transfer_encoding; + + proxy_pass http://app_server; + proxy_read_timeout 2000; + proxy_temp_path /mnt/logs/proxy_temp 1 2; +} \ No newline at end of file diff --git a/config.py b/config.py index 90705d337..55492bab1 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ import logging -import sys +import os +import logstash_formatter from peewee import MySQLDatabase, SqliteDatabase from storage.s3 import S3Storage @@ -11,10 +12,6 @@ from test.teststorage import FakeStorage, FakeUserfiles from test import analytics as fake_analytics -LOG_FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - ' + \ - '%(funcName)s - %(message)s' - - class FlaskConfig(object): SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884' @@ -138,12 +135,26 @@ class BuildNodeConfig(object): BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G' +def logs_init_builder(level=logging.DEBUG, logfile=None): + @staticmethod + def init_logs(): + if logfile: + handler = logging.FileHandler(logfile) + else: + handler = logging.StreamHandler() + + root_logger = logging.getLogger('') + root_logger.setLevel(level) + formatter = logstash_formatter.LogstashFormatter() + handler.setFormatter(formatter) + root_logger.addHandler(handler) + + return init_logs + + class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, FakeAnalytics, StripeTestConfig): - LOGGING_CONFIG = { - 'level': logging.WARN, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder(logging.WARN) POPULATE_DB_TEST_DATA = True TESTING = True INCLUDE_TEST_ENDPOINTS = True @@ -152,10 +163,7 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'level': logging.DEBUG, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True INCLUDE_TEST_ENDPOINTS = True @@ -165,10 +173,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'level': logging.DEBUG, - 'format': LOG_FORMAT - } + LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 @@ -176,10 +181,6 @@ class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles): - LOGGING_CONFIG = { - 'stream': sys.stderr, - 'level': logging.DEBUG, - 'format': LOG_FORMAT, - 'filename': 'application.log', - } + + LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log') SEND_FILE_MAX_AGE_DEFAULT = 0 diff --git a/gunicorn_config.py b/gunicorn_config.py index 59141b06d..88055c1fa 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -3,3 +3,4 @@ workers = 8 worker_class = 'gevent' timeout = 2000 daemon = True +pidfile = '/mnt/logs/gunicorn.pid' \ No newline at end of file diff --git a/initdb.py b/initdb.py index d00c0eeb9..0cdb5f91a 100644 --- a/initdb.py +++ b/initdb.py @@ -388,7 +388,7 @@ def populate_database(): metadata={'token_code': 'somecode', 'repo': 'orgrepo'}) if __name__ == '__main__': - logging.basicConfig(**app.config['LOGGING_CONFIG']) + app.config['LOGGING_CONFIG']() initialize_database() if app.config.get('POPULATE_DB_TEST_DATA', False): diff --git a/nginx-staging.conf b/nginx-staging.conf deleted file mode 100644 index b29bda2c5..000000000 --- a/nginx-staging.conf +++ /dev/null @@ -1,83 +0,0 @@ -worker_processes 2; - -user root nogroup; -pid /mnt/nginx/nginx.pid; -error_log /mnt/nginx/nginx.error.log; - -events { - worker_connections 1024; - accept_mutex off; -} - -http { - types_hash_max_size 2048; - include /usr/local/nginx/conf/mime.types.default; - - default_type application/octet-stream; - access_log /mnt/nginx/nginx.access.log combined; - sendfile on; - - root /root/quay/; - - gzip on; - gzip_http_version 1.0; - gzip_proxied any; - gzip_min_length 500; - gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/xml text/css - text/javascript application/x-javascript - application/octet-stream; - - upstream app_server { - server unix:/tmp/gunicorn.sock fail_timeout=0; - # For a TCP configuration: - # server 192.168.0.7:8000 fail_timeout=0; - } - - server { - listen 80 default_server; - server_name _; - rewrite ^ https://$host$request_uri? permanent; - } - - server { - listen 443 default; - client_max_body_size 8G; - client_body_temp_path /mnt/nginx/client_body 1 2; - server_name _; - - keepalive_timeout 5; - - ssl on; - ssl_certificate ./certs/quay-staging-unified.cert; - ssl_certificate_key ./certs/quay-staging.key; - ssl_session_timeout 5m; - ssl_protocols SSLv3 TLSv1; - ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; - ssl_prefer_server_ciphers on; - - if ($args ~ "_escaped_fragment_") { - rewrite ^ /snapshot$uri; - } - - location /static/ { - # checks for static file, if not found proxy to app - alias /root/quay/static/; - } - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_buffering off; - - proxy_request_buffering off; - proxy_set_header Transfer-Encoding $http_transfer_encoding; - - proxy_pass http://app_server; - proxy_read_timeout 2000; - proxy_temp_path /mnt/nginx/proxy_temp 1 2; - } - } -} diff --git a/nginx.conf b/nginx.conf deleted file mode 100644 index 53cd3c9f8..000000000 --- a/nginx.conf +++ /dev/null @@ -1,81 +0,0 @@ -worker_processes 8; - -user nobody nogroup; -pid /mnt/nginx/nginx.pid; -error_log /mnt/nginx/nginx.error.log; - -events { - worker_connections 1024; - accept_mutex off; -} - -http { - types_hash_max_size 2048; - include /usr/local/nginx/conf/mime.types.default; - - default_type application/octet-stream; - access_log /mnt/nginx/nginx.access.log combined; - sendfile on; - - gzip on; - gzip_http_version 1.0; - gzip_proxied any; - gzip_min_length 500; - gzip_disable "MSIE [1-6]\."; - gzip_types text/plain text/xml text/css - text/javascript application/x-javascript - application/octet-stream; - - upstream app_server { - server unix:/tmp/gunicorn.sock fail_timeout=0; - # For a TCP configuration: - # server 192.168.0.7:8000 fail_timeout=0; - } - - server { - listen 80 default_server; - server_name _; - rewrite ^ https://$host$request_uri? permanent; - } - - server { - listen 443 default; - client_max_body_size 8G; - client_body_temp_path /mnt/nginx/client_body 1 2; - server_name _; - - keepalive_timeout 5; - - ssl on; - ssl_certificate ./certs/quay-unified.cert; - ssl_certificate_key ./certs/quay.key; - ssl_session_timeout 5m; - ssl_protocols SSLv3 TLSv1; - ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; - ssl_prefer_server_ciphers on; - - if ($args ~ "_escaped_fragment_") { - rewrite ^ /snapshot$uri; - } - - location /static/ { - # checks for static file, if not found proxy to app - alias /home/ubuntu/quay/static/; - } - - location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_buffering off; - - proxy_request_buffering off; - proxy_set_header Transfer-Encoding $http_transfer_encoding; - - proxy_pass http://app_server; - proxy_read_timeout 2000; - proxy_temp_path /mnt/nginx/proxy_temp 1 2; - } - } -} \ No newline at end of file diff --git a/requirements-nover.txt b/requirements-nover.txt index f42a8a90d..c430edf5a 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -17,4 +17,5 @@ apscheduler python-daemon paramiko python-digitalocean -xhtml2pdf \ No newline at end of file +xhtml2pdf +logstash_formatter diff --git a/requirements.txt b/requirements.txt index 98697d7b6..7438e6dce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -APScheduler==2.1.1 +APScheduler==2.1.2 Flask==0.10.1 Flask-Login==0.2.9 Flask-Mail==0.9.0 Flask-Principal==0.4.0 -Jinja2==2.7.1 +Jinja2==2.7.2 MarkupSafe==0.18 Pillow==2.3.0 PyMySQL==0.6.1 @@ -11,18 +11,19 @@ Werkzeug==0.9.4 argparse==1.2.1 beautifulsoup4==4.3.2 blinker==1.3 -boto==2.21.2 +boto==2.24.0 distribute==0.6.34 ecdsa==0.10 gevent==1.0 -greenlet==0.4.1 +greenlet==0.4.2 gunicorn==18.0 html5lib==1.0b3 itsdangerous==0.23 lockfile==0.9.1 +logstash-formatter==0.5.8 marisa-trie==0.5.1 -mixpanel-py==3.0.0 -paramiko==1.12.0 +mixpanel-py==3.1.1 +paramiko==1.12.1 peewee==2.2.0 py-bcrypt==0.4 pyPdf==1.13 @@ -31,8 +32,8 @@ python-daemon==1.6 python-dateutil==2.2 python-digitalocean==0.6 reportlab==2.7 -requests==2.1.0 -six==1.4.1 -stripe==1.11.0 +requests==2.2.1 +six==1.5.2 +stripe==1.12.0 wsgiref==0.1.2 xhtml2pdf==0.0.5