From 8fac0474b5bef8eefcd5292b5d0665c27c3cac05 Mon Sep 17 00:00:00 2001 From: jakedt Date: Thu, 10 Apr 2014 18:30:09 -0400 Subject: [PATCH] Get staging to run under docker on an EC2 host. --- README.md | 28 +++++++++------------------- application.py | 3 ++- conf/nginx-enterprise.conf | 4 ++-- data/database.py | 22 +++++++++++----------- requirements-nover.txt | 4 +++- requirements.enterprise | 9 ++++++--- requirements.txt | 9 ++++++--- workers/README.md | 2 +- 8 files changed, 40 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 958070640..fdacaba51 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,35 @@ to prepare a new host: ``` +curl -s https://get.docker.io/ubuntu/ | sudo sh sudo apt-get update -sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core +sudo apt-get install -y git ``` -check out the code: +start the quay processes: ``` -git clone https://bitbucket.org/yackob03/quay.git -cd quay -virtualenv --distribute venv -source venv/bin/activate -pip install -r requirements.txt -sudo gdebi --n binary_dependencies/*.deb -sudo cp conf/logrotate/* /etc/logrotate.d/ -``` - -running: - -``` -sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && sudo /usr/local/nginx/sbin/nginx -c `pwd`/conf/nginx.conf -sudo mkdir -p /mnt/logs/ && sudo chown $USER /mnt/logs/ && STACK=prod gunicorn -c conf/gunicorn_config.py application:application +git clone git clone https://bitbucket.org/yackob03/quayconfig.git +sudo docker pull quay.io/quay/quay +sudo mkdir -p /mnt/logs/ +sudo docker run -d -p 80:80 -p 443:443 -v /mnt/logs:/mnt/logs -v `pwd`/quayconfig/production:/conf/stack quay.io/quay/quay ``` start the log shipper: ``` -curl -s https://get.docker.io/ubuntu/ | sudo sh sudo docker pull quay.io/quay/logstash sudo docker run -d -e REDIS_PORT_6379_TCP_ADDR=logs.quay.io -v /mnt/logs:/mnt/logs quay.io/quay/logstash quay.conf ``` -start the workers: +start the workers (FIXME): ``` STACK=prod python -m workers.diffsworker -D STACK=prod python -m workers.webhookworker -D ``` -bouncing the servers: +bouncing the servers (FIXME): ``` sudo kill -HUP `cat /mnt/logs/nginx.pid` diff --git a/application.py b/application.py index 91062d4f6..afc503379 100644 --- a/application.py +++ b/application.py @@ -1,11 +1,12 @@ import logging from app import app as application -from data.model import db as model_db # Initialize logging application.config['LOGGING_CONFIG']() +from data.model import db as model_db + # Turn off debug logging for boto logging.getLogger('boto').setLevel(logging.CRITICAL) diff --git a/conf/nginx-enterprise.conf b/conf/nginx-enterprise.conf index aa0d6c2c7..b704dc2db 100644 --- a/conf/nginx-enterprise.conf +++ b/conf/nginx-enterprise.conf @@ -15,8 +15,8 @@ http { listen 443 default; ssl on; - ssl_certificate ./ssl.cert; - ssl_certificate_key ./ssl.key; + ssl_certificate ./stack/ssl.cert; + ssl_certificate_key ./stack/ssl.key; ssl_session_timeout 5m; ssl_protocols SSLv3 TLSv1; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; diff --git a/data/database.py b/data/database.py index 219222b0d..6fe6760ec 100644 --- a/data/database.py +++ b/data/database.py @@ -5,6 +5,7 @@ import uuid from random import SystemRandom from datetime import datetime from peewee import * +from sqlalchemy.engine.url import make_url from urlparse import urlparse from app import app @@ -21,19 +22,18 @@ SCHEME_DRIVERS = { def generate_db(config_object): db_kwargs = dict(config_object['DB_CONNECTION_ARGS']) - connection_string = config_object['DB_URI'] + parsed_url = make_url(config_object['DB_URI']) - scheme, auth_and_host, dbname = urlparse(connection_string)[:3] - if auth_and_host: - if '@' in auth_and_host: - auth, db_kwargs['host'] = auth_and_host.split('@') - if ':' in auth: - db_kwargs['user'], db_kwargs['passwd'] = auth.split(':') + if parsed_url.host: + db_kwargs['host'] = parsed_url.host + if parsed_url.port: + db_kwargs['port'] = parsed_url.port + if parsed_url.username: + db_kwargs['user'] = parsed_url.username + if parsed_url.password: + db_kwargs['passwd'] = parsed_url.password - if scheme == 'sqlite': - dbname = dbname[1:] - - return SCHEME_DRIVERS[scheme](dbname, **db_kwargs) + return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs) db = generate_db(app.config) diff --git a/requirements-nover.txt b/requirements-nover.txt index 8bd0d7946..439bf714d 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -26,4 +26,6 @@ loremipsum pygithub flask-restful jsonschema -git+https://github.com/NateFerrero/oauth2lib.git \ No newline at end of file +git+https://github.com/NateFerrero/oauth2lib.git +alembic +sqlalchemy \ No newline at end of file diff --git a/requirements.enterprise b/requirements.enterprise index f4aaefca2..3101dfe0d 100644 --- a/requirements.enterprise +++ b/requirements.enterprise @@ -5,11 +5,14 @@ Flask-Mail==0.9.0 Flask-Principal==0.4.0 Flask-RESTful==0.2.12 Jinja2==2.7.2 +Mako==0.9.1 MarkupSafe==0.19 -Pillow==2.3.1 +Pillow==2.4.0 PyGithub==1.24.1 PyMySQL==0.6.1 +SQLAlchemy==0.9.4 Werkzeug==0.9.4 +alembic==0.6.4 aniso8601==0.82 argparse==1.2.1 beautifulsoup4==4.3.2 @@ -23,7 +26,7 @@ greenlet==0.4.2 gunicorn==18.0 hiredis==0.1.2 html5lib==1.0b3 -itsdangerous==0.23 +itsdangerous==0.24 jsonschema==2.3.0 lockfile==0.9.1 logstash-formatter==0.5.8 @@ -45,7 +48,7 @@ redis==2.9.1 reportlab==2.7 requests==2.2.1 six==1.6.1 -stripe==1.12.2 +stripe==1.14.0 websocket-client==0.11.0 wsgiref==0.1.2 xhtml2pdf==0.0.5 diff --git a/requirements.txt b/requirements.txt index 46def6790..3df9f03e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,11 +5,14 @@ Flask-Mail==0.9.0 Flask-Principal==0.4.0 Flask-RESTful==0.2.12 Jinja2==2.7.2 +Mako==0.9.1 MarkupSafe==0.19 -Pillow==2.3.1 +Pillow==2.4.0 PyGithub==1.24.1 PyMySQL==0.6.1 +SQLAlchemy==0.9.4 Werkzeug==0.9.4 +alembic==0.6.4 aniso8601==0.82 argparse==1.2.1 beautifulsoup4==4.3.2 @@ -23,7 +26,7 @@ greenlet==0.4.2 gunicorn==18.0 hiredis==0.1.2 html5lib==1.0b3 -itsdangerous==0.23 +itsdangerous==0.24 jsonschema==2.3.0 lockfile==0.9.1 logstash-formatter==0.5.8 @@ -45,7 +48,7 @@ redis==2.9.1 reportlab==2.7 requests==2.2.1 six==1.6.1 -stripe==1.12.2 +stripe==1.14.0 websocket-client==0.11.0 wsgiref==0.1.2 xhtml2pdf==0.0.5 diff --git a/workers/README.md b/workers/README.md index 8cee85eee..d4d2a3f14 100644 --- a/workers/README.md +++ b/workers/README.md @@ -2,7 +2,7 @@ to prepare a new build node host: ``` sudo apt-get update -sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core +sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core lxc ``` check out the code, install the kernel, custom docker, nsexec, and reboot: