diff --git a/README.md b/README.md index ebeee06ad..9bb6c3f9a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ running: ``` sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf -sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application +sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c conf/gunicorn_config.py application:application ``` start the log shipper: diff --git a/conf/cloud-init.sh b/conf/cloud-init.sh new file mode 100755 index 000000000..e3a69c302 --- /dev/null +++ b/conf/cloud-init.sh @@ -0,0 +1,60 @@ +#! /bin/sh + +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 +sh -c "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list" +apt-get update + +apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core lxc-docker + +PRIVATE_KEY=/root/.ssh/id_rsa +echo '-----BEGIN RSA PRIVATE KEY-----' > $PRIVATE_KEY +echo 'MIIEpAIBAAKCAQEA1qYjqPJAOHzE9jyE06LgOYFXtmVWMMPdS10oWUH77/M406/l' >> $PRIVATE_KEY +echo 'BG1Nf8VU2/q7VogfR/k56xumlAYcoEP9rueEMI9j2RwDy2s5SHaT7Z+9SyZnTRtq' >> $PRIVATE_KEY +echo 'bomTUHVBQtxgRXz2XHROWtFG54MhZtIHDk31kW2qyr+rMw2/kT1h6+s9D1mF5A5i' >> $PRIVATE_KEY +echo 'DWxNQSWYyS9gaM5a5aNUVscoXAtSG7JwY4XdYEGKXwMm7UYFeHlOPH/QRTZVO9XP' >> $PRIVATE_KEY +echo 'Z/vNW1t6JZ9GIAxfFP9v2YyehF3l2R+m3VGDld4JNosUPyWOnMPbHBcTYGe2nLgj' >> $PRIVATE_KEY +echo 'zH9mqhXKR0jR2hbo0QJz5ln8TXmj5v3mfPrF1QIDAQABAoIBAC52Y/2sAm63w0Kx' >> $PRIVATE_KEY +echo 'subEuNh5wOzAXrnLi9lGXveDKu+zrDdWObKNnlrr8gRz7505ddv0fK8BmzsrX4Lp' >> $PRIVATE_KEY +echo 'dL4paxm/0BMs1z1vBkVDNZ4YF7dupqmwJ4epy/N8jhXU8hnYhNNacaOC7WArqE1D' >> $PRIVATE_KEY +echo 'ZTeZdHB4VqHwfzRb432i1dFlaCAsEQ+pRg+o0wOqH5BMZy4LY5vESK5d2E85KhqT' >> $PRIVATE_KEY +echo '1rgD2T2FrkM42H4QvYzn6ntmjRAA5eO6RSeyPlkpniNTlmSuNYt8iqx8bm1HgXFn' >> $PRIVATE_KEY +echo 'Iova/9MifFt9CFG5SJPmYkPYvAEhNmiRdob68a/0BIX+Uuc1skX72Lpb/XjqrlXZ' >> $PRIVATE_KEY +echo 'UhJYALkCgYEA9fPGq9bGNWodCwplXuq5ydZv1BK5NZod+H85hUOz+gUN12UJ3Euy' >> $PRIVATE_KEY +echo 'FAZZqV5kwQ0i1cE6Vfg9SSk1V9osdw3TIVZgTOBKBYxsuCJzIO4zlyM7qi0XFsam' >> $PRIVATE_KEY +echo 'ax/v/kfHFnoBOPruJs0Ao5F4cGhZBfS4dQZAh4EqplSjJuGoLVMbNTsCgYEA32r8' >> $PRIVATE_KEY +echo 'kspbaCK71hDc2vAxVpHR3UNSui6lQCKOC4BbA8c1XP08+BKPONeNMaytXiRe5Vrq' >> $PRIVATE_KEY +echo 'bXRf9GqY6zzM08If78qjgDd2cfVYPnrb8unth7Z7QbsSi5+E6Gt8cevBEQqv1n6H' >> $PRIVATE_KEY +echo 'jzLKlETL5qpMpRHJi98AvyHcSpYyI6XORZE0AC8CgYEAwJJDPq5l+NKBtPBJ2Jxu' >> $PRIVATE_KEY +echo 'JUN5wZF7ZCWsS7HJZrdQxnSIltpscwjtgFJMh5j5yFGxsa2eMEuyKINUWdngMMMp' >> $PRIVATE_KEY +echo 'SRPpSKfgLSH6yd1nSSRYToDuqVqulk2pZXzXGsA2eDnElUmbh9PBKVCv/UsmUMyA' >> $PRIVATE_KEY +echo 'VFg11CLlMuBX8gyC8iH8zpsCgYB2NxDfxuzoxAApu5Bw1Ej26n9mGTpLw2Sy89W/' >> $PRIVATE_KEY +echo 'JjKCZETLKD+7b26TABL4psqxFoOTzjBerAYduM2jIu+qWHw3kDxFGpO0psIDhVSe' >> $PRIVATE_KEY +echo 'SsLhXWAInqiockaMCFu3l6v3jXUPBLJLxe9E1sYhDhkx+qBvPxcRCySZ3rE3BYOI' >> $PRIVATE_KEY +echo 'cdVXBwKBgQD1Wp1eLdnA3UV2KzyyVG3K/FqKszte70NfR9gvl6bD8cGeoAAt+iyW' >> $PRIVATE_KEY +echo 'Wd3tc3FKcDfywRoxrc4Atew0ySZVv78E2vDiyAswMhsbdldALw0sftaTIfdkXzlO' >> $PRIVATE_KEY +echo '77cUl9A2niF4mf0b8JeIGrTR81f3Q/ZRjzXMg/dZLVMtzPsFd9clGw==' >> $PRIVATE_KEY +echo '-----END RSA PRIVATE KEY-----' >> $PRIVATE_KEY +chmod 600 $PRIVATE_KEY + +BITBUCKET=AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== +KNOWN_HOSTS=/root/.ssh/known_hosts +echo "|1|7Yac4eoTmXJj7g7Hdlz0PdJMNnQ=|5AckfCb6pvVav45AOBMStvCVwFk= ssh-rsa $BITBUCKET" >> $KNOWN_HOSTS +echo "|1|epKB6bDLmj4UCWcN2lJ9NT+WjS4=|MThQkD3gLXsDEdRGD15uBlI6j5Q= ssh-rsa $BITBUCKET" >> $KNOWN_HOSTS +echo "|1|tET4d+sodv8Zk+m/JXHj3OWpyUU=|8lo5vpeKH6yiflQpV+aNEsSZBtw= ssh-rsa $BITBUCKET" >> $KNOWN_HOSTS + +export USER=ubuntu + +git clone git@bitbucket.org:yackob03/quay.git /home/$USER/quay +cd /home/$USER/quay +virtualenv --distribute venv +venv/bin/pip install -r requirements.txt +gdebi --n binary_dependencies/*.deb +cp conf/logrotate/* /etc/logrotate.d/ +chown -R $USER:$USER /home/$USER/quay + +mkdir -p /mnt/logs/ && chown $USER /mnt/logs/ && /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf +mkdir -p /mnt/logs/ && chown $USER /mnt/logs/ && STACK=prod sudo -u $USER -E venv/bin/gunicorn -c conf/gunicorn_config.py application:application + +echo '{"https://quay.io/v1/": {"auth": "cXVheStkZXBsb3k6OVkxUFg3RDNJRTRLUFNHQ0lBTEgxN0VNNVYzWlRNUDhDTk5ISk5YQVEyTkpHQVM0OEJESDhKMVBVT1o4NjlNTA==", "email": ""}}' > /root/.dockercfg +docker pull quay.io/quay/logstash +docker run -d -e REDIS_PORT_6379_TCP_ADDR=logs.quay.io -v /mnt/logs:/mnt/logs quay.io/quay/logstash quay.conf diff --git a/conf/deploy b/conf/deploy new file mode 100644 index 000000000..72bc1265a --- /dev/null +++ b/conf/deploy @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA1qYjqPJAOHzE9jyE06LgOYFXtmVWMMPdS10oWUH77/M406/l +BG1Nf8VU2/q7VogfR/k56xumlAYcoEP9rueEMI9j2RwDy2s5SHaT7Z+9SyZnTRtq +bomTUHVBQtxgRXz2XHROWtFG54MhZtIHDk31kW2qyr+rMw2/kT1h6+s9D1mF5A5i +DWxNQSWYyS9gaM5a5aNUVscoXAtSG7JwY4XdYEGKXwMm7UYFeHlOPH/QRTZVO9XP +Z/vNW1t6JZ9GIAxfFP9v2YyehF3l2R+m3VGDld4JNosUPyWOnMPbHBcTYGe2nLgj +zH9mqhXKR0jR2hbo0QJz5ln8TXmj5v3mfPrF1QIDAQABAoIBAC52Y/2sAm63w0Kx +subEuNh5wOzAXrnLi9lGXveDKu+zrDdWObKNnlrr8gRz7505ddv0fK8BmzsrX4Lp +dL4paxm/0BMs1z1vBkVDNZ4YF7dupqmwJ4epy/N8jhXU8hnYhNNacaOC7WArqE1D +ZTeZdHB4VqHwfzRb432i1dFlaCAsEQ+pRg+o0wOqH5BMZy4LY5vESK5d2E85KhqT +1rgD2T2FrkM42H4QvYzn6ntmjRAA5eO6RSeyPlkpniNTlmSuNYt8iqx8bm1HgXFn +Iova/9MifFt9CFG5SJPmYkPYvAEhNmiRdob68a/0BIX+Uuc1skX72Lpb/XjqrlXZ +UhJYALkCgYEA9fPGq9bGNWodCwplXuq5ydZv1BK5NZod+H85hUOz+gUN12UJ3Euy +FAZZqV5kwQ0i1cE6Vfg9SSk1V9osdw3TIVZgTOBKBYxsuCJzIO4zlyM7qi0XFsam +ax/v/kfHFnoBOPruJs0Ao5F4cGhZBfS4dQZAh4EqplSjJuGoLVMbNTsCgYEA32r8 +kspbaCK71hDc2vAxVpHR3UNSui6lQCKOC4BbA8c1XP08+BKPONeNMaytXiRe5Vrq +bXRf9GqY6zzM08If78qjgDd2cfVYPnrb8unth7Z7QbsSi5+E6Gt8cevBEQqv1n6H +jzLKlETL5qpMpRHJi98AvyHcSpYyI6XORZE0AC8CgYEAwJJDPq5l+NKBtPBJ2Jxu +JUN5wZF7ZCWsS7HJZrdQxnSIltpscwjtgFJMh5j5yFGxsa2eMEuyKINUWdngMMMp +SRPpSKfgLSH6yd1nSSRYToDuqVqulk2pZXzXGsA2eDnElUmbh9PBKVCv/UsmUMyA +VFg11CLlMuBX8gyC8iH8zpsCgYB2NxDfxuzoxAApu5Bw1Ej26n9mGTpLw2Sy89W/ +JjKCZETLKD+7b26TABL4psqxFoOTzjBerAYduM2jIu+qWHw3kDxFGpO0psIDhVSe +SsLhXWAInqiockaMCFu3l6v3jXUPBLJLxe9E1sYhDhkx+qBvPxcRCySZ3rE3BYOI +cdVXBwKBgQD1Wp1eLdnA3UV2KzyyVG3K/FqKszte70NfR9gvl6bD8cGeoAAt+iyW +Wd3tc3FKcDfywRoxrc4Atew0ySZVv78E2vDiyAswMhsbdldALw0sftaTIfdkXzlO +77cUl9A2niF4mf0b8JeIGrTR81f3Q/ZRjzXMg/dZLVMtzPsFd9clGw== +-----END RSA PRIVATE KEY----- diff --git a/conf/deploy.pub b/conf/deploy.pub new file mode 100644 index 000000000..27165697f --- /dev/null +++ b/conf/deploy.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWpiOo8kA4fMT2PITTouA5gVe2ZVYww91LXShZQfvv8zjTr+UEbU1/xVTb+rtWiB9H+TnrG6aUBhygQ/2u54Qwj2PZHAPLazlIdpPtn71LJmdNG2puiZNQdUFC3GBFfPZcdE5a0UbngyFm0gcOTfWRbarKv6szDb+RPWHr6z0PWYXkDmINbE1BJZjJL2Bozlrlo1RWxyhcC1IbsnBjhd1gQYpfAybtRgV4eU48f9BFNlU71c9n+81bW3oln0YgDF8U/2/ZjJ6EXeXZH6bdUYOV3gk2ixQ/JY6cw9scFxNgZ7acuCPMf2aqFcpHSNHaFujRAnPmWfxNeaPm/eZ8+sXV jake@coreserver diff --git a/conf/gunicorn_config.py b/conf/gunicorn_config.py new file mode 100644 index 000000000..c7332fe96 --- /dev/null +++ b/conf/gunicorn_config.py @@ -0,0 +1,10 @@ +bind = 'unix:/tmp/gunicorn.sock' +workers = 8 +worker_class = 'gevent' +timeout = 2000 +daemon = True +pidfile = '/mnt/logs/gunicorn.pid' +errorlog = '/mnt/logs/application.log' +loglevel = 'debug' +logger_class = 'util.glogger.LogstashLogger' +pythonpath = '.' \ No newline at end of file diff --git a/conf/gunicorn_local.py b/conf/gunicorn_local.py new file mode 100644 index 000000000..2a145fd98 --- /dev/null +++ b/conf/gunicorn_local.py @@ -0,0 +1,9 @@ +bind = '0.0.0.0:5000' +workers = 2 +worker_class = 'gevent' +timeout = 2000 +daemon = False +errorlog = '-' +loglevel = 'debug' +logger_class = 'util.glogger.LogstashLogger' +pythonpath = '.' \ No newline at end of file diff --git a/conf/logrotate/quay-logrotate b/conf/logrotate/quay-logrotate index 79fdc377d..1a6678639 100644 --- a/conf/logrotate/quay-logrotate +++ b/conf/logrotate/quay-logrotate @@ -8,7 +8,7 @@ create 644 root root postrotate - [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid` + kill -USR1 `cat /mnt/logs/nginx.pid` endscript } @@ -22,7 +22,7 @@ create 644 root root postrotate - [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid` + kill -USR1 `cat /mnt/logs/nginx.pid` endscript } @@ -36,6 +36,6 @@ create 644 ubuntu ubuntu postrotate - [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/gunicorn.pid` + kill -USR1 `cat /mnt/logs/gunicorn.pid` endscript } \ No newline at end of file diff --git a/config.py b/config.py index 7a772a119..9be2c86a6 100644 --- a/config.py +++ b/config.py @@ -140,14 +140,10 @@ class BuildNodeConfig(object): BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G' -def logs_init_builder(level=logging.DEBUG, logfile=None): +def logs_init_builder(level=logging.DEBUG): @staticmethod def init_logs(): - if logfile: - handler = logging.FileHandler(logfile) - else: - handler = logging.StreamHandler() - + handler = logging.StreamHandler() root_logger = logging.getLogger('') root_logger.setLevel(level) formatter = logstash_formatter.LogstashFormatter() @@ -188,5 +184,5 @@ class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles, RedisBuildLogs): - LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log') + LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 diff --git a/gunicorn_config.py b/gunicorn_config.py deleted file mode 100644 index 88055c1fa..000000000 --- a/gunicorn_config.py +++ /dev/null @@ -1,6 +0,0 @@ -bind = 'unix:/tmp/gunicorn.sock' -workers = 8 -worker_class = 'gevent' -timeout = 2000 -daemon = True -pidfile = '/mnt/logs/gunicorn.pid' \ No newline at end of file diff --git a/tools/sendconfirmemail.py b/tools/sendconfirmemail.py new file mode 100644 index 000000000..3f7a02d95 --- /dev/null +++ b/tools/sendconfirmemail.py @@ -0,0 +1,28 @@ +from app import stripe +from app import app + +from util.email import send_confirmation_email + +from data import model + +import argparse + +from flask import Flask, current_app +from flask_mail import Mail + +def sendConfirmation(username): + user = model.get_user(username) + if not user: + print 'No user found' + return + + + with app.app_context(): + code = model.create_confirm_email_code(user) + send_confirmation_email(user.username, user.email, code.code) + print 'Email sent to %s' % (user.email) + +parser = argparse.ArgumentParser(description='Sends a confirmation email') +parser.add_argument('username', help='The username') +args = parser.parse_args() +sendConfirmation(args.username) diff --git a/util/glogger.py b/util/glogger.py new file mode 100644 index 000000000..44303bfdb --- /dev/null +++ b/util/glogger.py @@ -0,0 +1,23 @@ +import logging +import logstash_formatter +import gunicorn.glogging + +from gunicorn import util + +class LogstashLogger(gunicorn.glogging.Logger): + def _set_handler(self, log, output, fmt): + # remove previous gunicorn log handler + h = self._get_gunicorn_handler(log) + if h: + log.handlers.remove(h) + + if output is not None: + if output == "-": + h = logging.StreamHandler() + else: + util.check_is_writeable(output) + h = logging.FileHandler(output) + + h.setFormatter(logstash_formatter.LogstashFormatter()) + h._gunicorn = True + log.addHandler(h)