Merge branch 'master' into bobthe

README.md

@@ -21,7 +21,7 @@ running:
 
 ```
 sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf
-sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c gunicorn_config.py application:application
+sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c conf/gunicorn_config.py application:application
 ```
 
 start the log shipper:

conf/cloud-init.sh

@@ -0,0 +1,60 @@
+#! /bin/sh
+
+apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
+sh -c "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
+apt-get update
+
+apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core lxc-docker
+
+PRIVATE_KEY=/root/.ssh/id_rsa
+echo '-----BEGIN RSA PRIVATE KEY-----' > $PRIVATE_KEY
+echo 'MIIEpAIBAAKCAQEA1qYjqPJAOHzE9jyE06LgOYFXtmVWMMPdS10oWUH77/M406/l' >> $PRIVATE_KEY
+echo 'BG1Nf8VU2/q7VogfR/k56xumlAYcoEP9rueEMI9j2RwDy2s5SHaT7Z+9SyZnTRtq' >> $PRIVATE_KEY
+echo 'bomTUHVBQtxgRXz2XHROWtFG54MhZtIHDk31kW2qyr+rMw2/kT1h6+s9D1mF5A5i' >> $PRIVATE_KEY
+echo 'DWxNQSWYyS9gaM5a5aNUVscoXAtSG7JwY4XdYEGKXwMm7UYFeHlOPH/QRTZVO9XP' >> $PRIVATE_KEY
+echo 'Z/vNW1t6JZ9GIAxfFP9v2YyehF3l2R+m3VGDld4JNosUPyWOnMPbHBcTYGe2nLgj' >> $PRIVATE_KEY
+echo 'zH9mqhXKR0jR2hbo0QJz5ln8TXmj5v3mfPrF1QIDAQABAoIBAC52Y/2sAm63w0Kx' >> $PRIVATE_KEY
+echo 'subEuNh5wOzAXrnLi9lGXveDKu+zrDdWObKNnlrr8gRz7505ddv0fK8BmzsrX4Lp' >> $PRIVATE_KEY
+echo 'dL4paxm/0BMs1z1vBkVDNZ4YF7dupqmwJ4epy/N8jhXU8hnYhNNacaOC7WArqE1D' >> $PRIVATE_KEY
+echo 'ZTeZdHB4VqHwfzRb432i1dFlaCAsEQ+pRg+o0wOqH5BMZy4LY5vESK5d2E85KhqT' >> $PRIVATE_KEY
+echo '1rgD2T2FrkM42H4QvYzn6ntmjRAA5eO6RSeyPlkpniNTlmSuNYt8iqx8bm1HgXFn' >> $PRIVATE_KEY
+echo 'Iova/9MifFt9CFG5SJPmYkPYvAEhNmiRdob68a/0BIX+Uuc1skX72Lpb/XjqrlXZ' >> $PRIVATE_KEY
+echo 'UhJYALkCgYEA9fPGq9bGNWodCwplXuq5ydZv1BK5NZod+H85hUOz+gUN12UJ3Euy' >> $PRIVATE_KEY
+echo 'FAZZqV5kwQ0i1cE6Vfg9SSk1V9osdw3TIVZgTOBKBYxsuCJzIO4zlyM7qi0XFsam' >> $PRIVATE_KEY
+echo 'ax/v/kfHFnoBOPruJs0Ao5F4cGhZBfS4dQZAh4EqplSjJuGoLVMbNTsCgYEA32r8' >> $PRIVATE_KEY
+echo 'kspbaCK71hDc2vAxVpHR3UNSui6lQCKOC4BbA8c1XP08+BKPONeNMaytXiRe5Vrq' >> $PRIVATE_KEY
+echo 'bXRf9GqY6zzM08If78qjgDd2cfVYPnrb8unth7Z7QbsSi5+E6Gt8cevBEQqv1n6H' >> $PRIVATE_KEY
+echo 'jzLKlETL5qpMpRHJi98AvyHcSpYyI6XORZE0AC8CgYEAwJJDPq5l+NKBtPBJ2Jxu' >> $PRIVATE_KEY
+echo 'JUN5wZF7ZCWsS7HJZrdQxnSIltpscwjtgFJMh5j5yFGxsa2eMEuyKINUWdngMMMp' >> $PRIVATE_KEY
+echo 'SRPpSKfgLSH6yd1nSSRYToDuqVqulk2pZXzXGsA2eDnElUmbh9PBKVCv/UsmUMyA' >> $PRIVATE_KEY
+echo 'VFg11CLlMuBX8gyC8iH8zpsCgYB2NxDfxuzoxAApu5Bw1Ej26n9mGTpLw2Sy89W/' >> $PRIVATE_KEY
+echo 'JjKCZETLKD+7b26TABL4psqxFoOTzjBerAYduM2jIu+qWHw3kDxFGpO0psIDhVSe' >> $PRIVATE_KEY
+echo 'SsLhXWAInqiockaMCFu3l6v3jXUPBLJLxe9E1sYhDhkx+qBvPxcRCySZ3rE3BYOI' >> $PRIVATE_KEY
+echo 'cdVXBwKBgQD1Wp1eLdnA3UV2KzyyVG3K/FqKszte70NfR9gvl6bD8cGeoAAt+iyW' >> $PRIVATE_KEY
+echo 'Wd3tc3FKcDfywRoxrc4Atew0ySZVv78E2vDiyAswMhsbdldALw0sftaTIfdkXzlO' >> $PRIVATE_KEY
+echo '77cUl9A2niF4mf0b8JeIGrTR81f3Q/ZRjzXMg/dZLVMtzPsFd9clGw==' >> $PRIVATE_KEY
+echo '-----END RSA PRIVATE KEY-----' >> $PRIVATE_KEY
+chmod 600 $PRIVATE_KEY
+
+BITBUCKET=AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw==
+KNOWN_HOSTS=/root/.ssh/known_hosts
+echo "|1|7Yac4eoTmXJj7g7Hdlz0PdJMNnQ=|5AckfCb6pvVav45AOBMStvCVwFk= ssh-rsa $BITBUCKET" >> $KNOWN_HOSTS
+echo "|1|epKB6bDLmj4UCWcN2lJ9NT+WjS4=|MThQkD3gLXsDEdRGD15uBlI6j5Q= ssh-rsa $BITBUCKET" >> $KNOWN_HOSTS
+echo "|1|tET4d+sodv8Zk+m/JXHj3OWpyUU=|8lo5vpeKH6yiflQpV+aNEsSZBtw= ssh-rsa $BITBUCKET" >> $KNOWN_HOSTS
+
+export USER=ubuntu
+
+git clone git@bitbucket.org:yackob03/quay.git /home/$USER/quay
+cd /home/$USER/quay
+virtualenv --distribute venv
+venv/bin/pip install -r requirements.txt
+gdebi --n binary_dependencies/*.deb
+cp conf/logrotate/* /etc/logrotate.d/
+chown -R $USER:$USER /home/$USER/quay
+
+mkdir -p /mnt/logs/ && chown $USER /mnt/logs/ && /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf
+mkdir -p /mnt/logs/ && chown $USER /mnt/logs/ && STACK=prod sudo -u $USER -E venv/bin/gunicorn -c conf/gunicorn_config.py application:application
+
+echo '{"https://quay.io/v1/": {"auth": "cXVheStkZXBsb3k6OVkxUFg3RDNJRTRLUFNHQ0lBTEgxN0VNNVYzWlRNUDhDTk5ISk5YQVEyTkpHQVM0OEJESDhKMVBVT1o4NjlNTA==", "email": ""}}' > /root/.dockercfg
+docker pull quay.io/quay/logstash
+docker run -d -e REDIS_PORT_6379_TCP_ADDR=logs.quay.io -v /mnt/logs:/mnt/logs quay.io/quay/logstash quay.conf

conf/deploy

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA1qYjqPJAOHzE9jyE06LgOYFXtmVWMMPdS10oWUH77/M406/l
+BG1Nf8VU2/q7VogfR/k56xumlAYcoEP9rueEMI9j2RwDy2s5SHaT7Z+9SyZnTRtq
+bomTUHVBQtxgRXz2XHROWtFG54MhZtIHDk31kW2qyr+rMw2/kT1h6+s9D1mF5A5i
+DWxNQSWYyS9gaM5a5aNUVscoXAtSG7JwY4XdYEGKXwMm7UYFeHlOPH/QRTZVO9XP
+Z/vNW1t6JZ9GIAxfFP9v2YyehF3l2R+m3VGDld4JNosUPyWOnMPbHBcTYGe2nLgj
+zH9mqhXKR0jR2hbo0QJz5ln8TXmj5v3mfPrF1QIDAQABAoIBAC52Y/2sAm63w0Kx
+subEuNh5wOzAXrnLi9lGXveDKu+zrDdWObKNnlrr8gRz7505ddv0fK8BmzsrX4Lp
+dL4paxm/0BMs1z1vBkVDNZ4YF7dupqmwJ4epy/N8jhXU8hnYhNNacaOC7WArqE1D
+ZTeZdHB4VqHwfzRb432i1dFlaCAsEQ+pRg+o0wOqH5BMZy4LY5vESK5d2E85KhqT
+1rgD2T2FrkM42H4QvYzn6ntmjRAA5eO6RSeyPlkpniNTlmSuNYt8iqx8bm1HgXFn
+Iova/9MifFt9CFG5SJPmYkPYvAEhNmiRdob68a/0BIX+Uuc1skX72Lpb/XjqrlXZ
+UhJYALkCgYEA9fPGq9bGNWodCwplXuq5ydZv1BK5NZod+H85hUOz+gUN12UJ3Euy
+FAZZqV5kwQ0i1cE6Vfg9SSk1V9osdw3TIVZgTOBKBYxsuCJzIO4zlyM7qi0XFsam
+ax/v/kfHFnoBOPruJs0Ao5F4cGhZBfS4dQZAh4EqplSjJuGoLVMbNTsCgYEA32r8
+kspbaCK71hDc2vAxVpHR3UNSui6lQCKOC4BbA8c1XP08+BKPONeNMaytXiRe5Vrq
+bXRf9GqY6zzM08If78qjgDd2cfVYPnrb8unth7Z7QbsSi5+E6Gt8cevBEQqv1n6H
+jzLKlETL5qpMpRHJi98AvyHcSpYyI6XORZE0AC8CgYEAwJJDPq5l+NKBtPBJ2Jxu
+JUN5wZF7ZCWsS7HJZrdQxnSIltpscwjtgFJMh5j5yFGxsa2eMEuyKINUWdngMMMp
+SRPpSKfgLSH6yd1nSSRYToDuqVqulk2pZXzXGsA2eDnElUmbh9PBKVCv/UsmUMyA
+VFg11CLlMuBX8gyC8iH8zpsCgYB2NxDfxuzoxAApu5Bw1Ej26n9mGTpLw2Sy89W/
+JjKCZETLKD+7b26TABL4psqxFoOTzjBerAYduM2jIu+qWHw3kDxFGpO0psIDhVSe
+SsLhXWAInqiockaMCFu3l6v3jXUPBLJLxe9E1sYhDhkx+qBvPxcRCySZ3rE3BYOI
+cdVXBwKBgQD1Wp1eLdnA3UV2KzyyVG3K/FqKszte70NfR9gvl6bD8cGeoAAt+iyW
+Wd3tc3FKcDfywRoxrc4Atew0ySZVv78E2vDiyAswMhsbdldALw0sftaTIfdkXzlO
+77cUl9A2niF4mf0b8JeIGrTR81f3Q/ZRjzXMg/dZLVMtzPsFd9clGw==
+-----END RSA PRIVATE KEY-----

conf/deploy.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWpiOo8kA4fMT2PITTouA5gVe2ZVYww91LXShZQfvv8zjTr+UEbU1/xVTb+rtWiB9H+TnrG6aUBhygQ/2u54Qwj2PZHAPLazlIdpPtn71LJmdNG2puiZNQdUFC3GBFfPZcdE5a0UbngyFm0gcOTfWRbarKv6szDb+RPWHr6z0PWYXkDmINbE1BJZjJL2Bozlrlo1RWxyhcC1IbsnBjhd1gQYpfAybtRgV4eU48f9BFNlU71c9n+81bW3oln0YgDF8U/2/ZjJ6EXeXZH6bdUYOV3gk2ixQ/JY6cw9scFxNgZ7acuCPMf2aqFcpHSNHaFujRAnPmWfxNeaPm/eZ8+sXV jake@coreserver

conf/gunicorn_config.py

@@ -0,0 +1,10 @@
+bind = 'unix:/tmp/gunicorn.sock'
+workers = 8
+worker_class = 'gevent'
+timeout = 2000
+daemon = True
+pidfile = '/mnt/logs/gunicorn.pid'
+errorlog = '/mnt/logs/application.log'
+loglevel = 'debug'
+logger_class = 'util.glogger.LogstashLogger'
+pythonpath = '.'

conf/gunicorn_local.py

@@ -0,0 +1,9 @@
+bind = '0.0.0.0:5000'
+workers = 2
+worker_class = 'gevent'
+timeout = 2000
+daemon = False
+errorlog = '-'
+loglevel = 'debug'
+logger_class = 'util.glogger.LogstashLogger'
+pythonpath = '.'

conf/logrotate/quay-logrotate

@@ -8,7 +8,7 @@
   create 644 root root
 
   postrotate
-    [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid`
+    kill -USR1 `cat /mnt/logs/nginx.pid`
   endscript
 }
 
@@ -22,7 +22,7 @@
   create 644 root root
 
   postrotate
-    [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/nginx.pid`
+    kill -USR1 `cat /mnt/logs/nginx.pid`
   endscript
 }
 
@@ -36,6 +36,6 @@
   create 644 ubuntu ubuntu
 
   postrotate
-    [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /mnt/logs/gunicorn.pid`
+    kill -USR1 `cat /mnt/logs/gunicorn.pid`
   endscript
 }

config.py

@@ -140,14 +140,10 @@ class BuildNodeConfig(object):
   BUILD_NODE_PULL_TOKEN = 'F02O2E86CQLKZUQ0O81J8XDHQ6F0N1V36L9JTOEEK6GKKMT1GI8PTJQT4OU88Y6G'
 
 
-def logs_init_builder(level=logging.DEBUG, logfile=None):
+def logs_init_builder(level=logging.DEBUG):
   @staticmethod
   def init_logs():
-    if logfile:
-      handler = logging.FileHandler(logfile)
-    else:
-      handler = logging.StreamHandler()
-
+    handler = logging.StreamHandler()
     root_logger = logging.getLogger('')
     root_logger.setLevel(level)
     formatter = logstash_formatter.LogstashFormatter()
@@ -188,5 +184,5 @@ class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL,
                        GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig,
                        S3Userfiles, RedisBuildLogs):
 
-  LOGGING_CONFIG = logs_init_builder(logfile='/mnt/logs/application.log')
+  LOGGING_CONFIG = logs_init_builder()
   SEND_FILE_MAX_AGE_DEFAULT = 0

gunicorn_config.py

@@ -1,6 +0,0 @@
-bind = 'unix:/tmp/gunicorn.sock'
-workers = 8
-worker_class = 'gevent'
-timeout = 2000
-daemon = True
-pidfile = '/mnt/logs/gunicorn.pid'

tools/sendconfirmemail.py

@@ -0,0 +1,28 @@
+from app import stripe
+from app import app
+
+from util.email import send_confirmation_email
+
+from data import model
+
+import argparse
+
+from flask import Flask, current_app
+from flask_mail import Mail
+
+def sendConfirmation(username):
+  user = model.get_user(username)
+  if not user:
+    print 'No user found'
+    return
+
+
+  with app.app_context():
+    code = model.create_confirm_email_code(user)
+    send_confirmation_email(user.username, user.email, code.code)
+    print 'Email sent to %s' % (user.email)
+
+parser = argparse.ArgumentParser(description='Sends a confirmation email')
+parser.add_argument('username', help='The username')
+args = parser.parse_args()
+sendConfirmation(args.username)

util/glogger.py

@@ -0,0 +1,23 @@
+import logging
+import logstash_formatter
+import gunicorn.glogging
+
+from gunicorn import util
+
+class LogstashLogger(gunicorn.glogging.Logger):
+  def _set_handler(self, log, output, fmt):
+    # remove previous gunicorn log handler
+    h = self._get_gunicorn_handler(log)
+    if h:
+      log.handlers.remove(h)
+
+    if output is not None:
+      if output == "-":
+        h = logging.StreamHandler()
+      else:
+        util.check_is_writeable(output)
+        h = logging.FileHandler(output)
+
+      h.setFormatter(logstash_formatter.LogstashFormatter())
+      h._gunicorn = True
+      log.addHandler(h)