diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e5a9d7e07..000000000 --- a/Dockerfile +++ /dev/null @@ -1,72 +0,0 @@ -FROM phusion/baseimage:0.9.9 - -ENV DEBIAN_FRONTEND noninteractive -ENV HOME /root - -# Needed for this fix: http://stackoverflow.com/a/21715730 -RUN apt-get update -RUN apt-get install -y software-properties-common python-software-properties -RUN add-apt-repository ppa:chris-lea/node.js - -# Install the dependencies. -RUN apt-get update - -# New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 libssl1.0.0 - -# PhantomJS -RUN apt-get install -y libfreetype6 libfreetype6-dev fontconfig -ADD https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 phantomjs.tar.bz2 -RUN tar xjf phantomjs.tar.bz2 && ln -s `pwd`/phantomjs*/bin/phantomjs /usr/bin/phantomjs - -# Grunt -RUN apt-get install -y nodejs -RUN npm install -g grunt-cli - -ADD binary_dependencies binary_dependencies -RUN gdebi --n binary_dependencies/*.deb - -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -ADD requirements.txt requirements.txt -RUN virtualenv --distribute venv -RUN venv/bin/pip install -r requirements.txt - -ADD auth auth -ADD buildstatus buildstatus -ADD conf conf -ADD data data -ADD endpoints endpoints -ADD features features -ADD grunt grunt -ADD screenshots screenshots -ADD static static -ADD storage storage -ADD templates templates -ADD util util -ADD workers workers - -ADD app.py app.py -ADD application.py application.py -ADD config.py config.py -ADD initdb.py initdb.py - -ADD conf/init/mklogsdir.sh /etc/my_init.d/ -ADD conf/init/gunicorn.sh /etc/service/gunicorn/run -ADD conf/init/nginx.sh /etc/service/nginx/run -ADD conf/init/diffsworker.sh /etc/service/diffsworker/run -ADD conf/init/webhookworker.sh /etc/service/webhookworker/run - -RUN cd grunt && npm install -RUN cd grunt && grunt - -# Add the tests last because they're prone to accidental changes, then run them -ADD test test -RUN TEST=true venv/bin/python -m unittest discover - -RUN rm -rf /conf/stack -VOLUME ["/conf/stack", "/mnt/logs"] - -EXPOSE 443 80 - -CMD ["/sbin/my_init"] \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 120000 index 000000000..40c87ea59 --- /dev/null +++ b/Dockerfile @@ -0,0 +1 @@ +Dockerfile.web \ No newline at end of file diff --git a/Dockerfile.buildworker b/Dockerfile.buildworker new file mode 100644 index 000000000..4ad6ba6ff --- /dev/null +++ b/Dockerfile.buildworker @@ -0,0 +1,48 @@ +FROM phusion/baseimage:0.9.10 + +ENV DEBIAN_FRONTEND noninteractive +ENV HOME /root + +RUN apt-get update +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 + +### End common section ### + +RUN apt-get install -y lxc aufs-tools + +RUN usermod -v 100000-200000 -w 100000-200000 root + +ADD binary_dependencies/builder binary_dependencies/builder + +RUN gdebi --n binary_dependencies/builder/*.deb + +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ADD requirements.txt requirements.txt +RUN virtualenv --distribute venv +RUN venv/bin/pip install -r requirements.txt + +ADD buildstatus buildstatus +ADD data data +ADD features features +ADD storage storage +ADD util util +ADD workers workers + +ADD app.py app.py +ADD config.py config.py + +# Remove this if we ever stop depending on test data for the default config +ADD test test + +ADD conf conf +RUN rm -rf /conf/stack + +ADD conf/init/svlogd_config /svlogd_config +ADD conf/init/preplogsdir.sh /etc/my_init.d/ +ADD conf/init/tutumdocker /etc/service/tutumdocker +ADD conf/init/dockerfilebuild /etc/service/dockerfilebuild + +VOLUME ["/var/lib/docker", "/var/lib/lxc", "/conf/stack", "/var/log"] + +CMD ["/sbin/my_init"] \ No newline at end of file diff --git a/Dockerfile.web b/Dockerfile.web new file mode 100644 index 000000000..e4958e6e0 --- /dev/null +++ b/Dockerfile.web @@ -0,0 +1,67 @@ +FROM phusion/baseimage:0.9.10 + +ENV DEBIAN_FRONTEND noninteractive +ENV HOME /root + +# Install the dependencies. +RUN apt-get update + +# New ubuntu packages should be added as their own apt-get install lines below the existing install commands +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62-dev libevent-dev gdebi-core g++ libmagic1 + +# PhantomJS +RUN apt-get install -y phantomjs + +# Grunt +RUN apt-get install -y nodejs npm +RUN ln -s /usr/bin/nodejs /usr/bin/node +RUN npm install -g grunt-cli + +ADD binary_dependencies binary_dependencies +RUN gdebi --n binary_dependencies/*.deb + +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ADD requirements.txt requirements.txt +RUN virtualenv --distribute venv +RUN venv/bin/pip install -r requirements.txt + +ADD auth auth +ADD buildstatus buildstatus +ADD conf conf +ADD data data +ADD endpoints endpoints +ADD features features +ADD grunt grunt +ADD screenshots screenshots +ADD static static +ADD storage storage +ADD templates templates +ADD util util +ADD workers workers + +ADD app.py app.py +ADD application.py application.py +ADD config.py config.py +ADD initdb.py initdb.py + +ADD conf/init/svlogd_config /svlogd_config +ADD conf/init/preplogsdir.sh /etc/my_init.d/ +ADD conf/init/gunicorn /etc/service/gunicorn +ADD conf/init/nginx /etc/service/nginx +ADD conf/init/diffsworker /etc/service/diffsworker +ADD conf/init/webhookworker /etc/service/webhookworker + +RUN cd grunt && npm install +RUN cd grunt && grunt + +# Add the tests last because they're prone to accidental changes, then run them +ADD test test +RUN TEST=true venv/bin/python -m unittest discover + +RUN rm -rf /conf/stack +VOLUME ["/conf/stack", "/var/log"] + +EXPOSE 443 80 + +CMD ["/sbin/my_init"] \ No newline at end of file diff --git a/README.md b/README.md index f80bea5e7..61cd823ad 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,10 @@ to build and upload quay to quay: ``` curl -s https://get.docker.io/ubuntu/ | sudo sh sudo apt-get update && sudo apt-get install -y git -git clone git clone https://bitbucket.org/yackob03/quay.git +git clone https://bitbucket.org/yackob03/quay.git cd quay +rm Dockerfile +ln -s Dockerfile.web Dockerfile sudo docker build -t quay.io/quay/quay . sudo docker push quay.io/quay/quay ``` @@ -19,7 +21,7 @@ cd gantryd cat requirements.system | xargs sudo apt-get install -y virtualenv --distribute venv venv/bin/pip install -r requirements.txt -sudo docker login -p 9Y1PX7D3IE4KPSGCIALH17EM5V3ZTMP8CNNHJNXAQ2NJGAS48BDH8J1PUOZ869ML -u 'quay+deploy' -e notused quay.io +sudo docker login -p 9Y1PX7D3IE4KPSGCIALH17EM5V3ZTMP8CNNHJNXAQ2NJGAS48BDH8J1PUOZ869ML -u 'quay+deploy' -e notused staging.quay.io ``` start the quay processes: @@ -27,8 +29,7 @@ start the quay processes: ``` cd ~ git clone https://bitbucket.org/yackob03/quayconfig.git -sudo docker pull quay.io/quay/quay -sudo mkdir -p /mnt/logs/ +sudo docker pull staging.quay.io/quay/quay cd ~/gantryd sudo venv/bin/python gantry.py ../quayconfig/production/gantry.json update quay ``` diff --git a/app.py b/app.py index aa663418c..f1b029b55 100644 --- a/app.py +++ b/app.py @@ -13,6 +13,7 @@ from data.userfiles import Userfiles from util.analytics import Analytics from util.exceptionlog import Sentry from data.billing import Billing +from data.buildlogs import BuildLogs OVERRIDE_CONFIG_FILENAME = 'conf/stack/config.py' @@ -46,3 +47,4 @@ userfiles = Userfiles(app) analytics = Analytics(app) billing = Billing(app) sentry = Sentry(app) +build_logs = BuildLogs(app) diff --git a/application.py b/application.py index e7cb11c76..32491d1f9 100644 --- a/application.py +++ b/application.py @@ -67,5 +67,5 @@ application.teardown_request(close_db) application.request_class = RequestWithId if __name__ == '__main__': - logging.config.fileConfig('conf/logging_local.conf', disable_existing_loggers=False) + logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) application.run(port=5000, debug=True, threaded=True, host='0.0.0.0') diff --git a/binary_dependencies/builder/lxc-docker-0.11.1-tutum2_0.11.1-tutum2-20140520173012-5a3b101-dirty_amd64.deb b/binary_dependencies/builder/lxc-docker-0.11.1-tutum2_0.11.1-tutum2-20140520173012-5a3b101-dirty_amd64.deb new file mode 100644 index 000000000..3e1ae44bb Binary files /dev/null and b/binary_dependencies/builder/lxc-docker-0.11.1-tutum2_0.11.1-tutum2-20140520173012-5a3b101-dirty_amd64.deb differ diff --git a/binary_dependencies/builder/lxc-docker-0.9.0_0.9.0-20140501212101-72572f0-dirty_amd64.deb b/binary_dependencies/builder/lxc-docker-0.9.0_0.9.0-20140501212101-72572f0-dirty_amd64.deb deleted file mode 100644 index 2242c23ce..000000000 Binary files a/binary_dependencies/builder/lxc-docker-0.9.0_0.9.0-20140501212101-72572f0-dirty_amd64.deb and /dev/null differ diff --git a/binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb b/binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb deleted file mode 100644 index e78b16986..000000000 Binary files a/binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb and /dev/null differ diff --git a/binary_dependencies/nginx_1.4.2-nobuffer-2_amd64.deb b/binary_dependencies/nginx_1.4.2-nobuffer-2_amd64.deb deleted file mode 100644 index dfc530e7f..000000000 Binary files a/binary_dependencies/nginx_1.4.2-nobuffer-2_amd64.deb and /dev/null differ diff --git a/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb b/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb new file mode 100644 index 000000000..8b3fb3fb6 Binary files /dev/null and b/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb differ diff --git a/conf/gunicorn_config.py b/conf/gunicorn_config.py index b86250125..4d9d50499 100644 --- a/conf/gunicorn_config.py +++ b/conf/gunicorn_config.py @@ -2,6 +2,6 @@ bind = 'unix:/tmp/gunicorn.sock' workers = 8 worker_class = 'gevent' timeout = 2000 -pidfile = '/tmp/gunicorn.pid' logconfig = 'conf/logging.conf' -pythonpath = '.' \ No newline at end of file +pythonpath = '.' +preload_app = True \ No newline at end of file diff --git a/conf/gunicorn_local.py b/conf/gunicorn_local.py index 9f93eb008..3d2a36844 100644 --- a/conf/gunicorn_local.py +++ b/conf/gunicorn_local.py @@ -3,5 +3,5 @@ workers = 2 worker_class = 'gevent' timeout = 2000 daemon = False -logconfig = 'conf/logging_local.conf' +logconfig = 'conf/logging.conf' pythonpath = '.' \ No newline at end of file diff --git a/conf/http-base.conf b/conf/http-base.conf index 32e8b3730..bfa1a85f2 100644 --- a/conf/http-base.conf +++ b/conf/http-base.conf @@ -1,20 +1,8 @@ -log_format logstash_json '{ "@timestamp": "$time_iso8601", ' - '"@fields": { ' - '"remote_addr": "$remote_addr", ' - '"remote_user": "$remote_user", ' - '"body_bytes_sent": "$body_bytes_sent", ' - '"request_time": "$request_time", ' - '"status": "$status", ' - '"request": "$request", ' - '"request_method": "$request_method", ' - '"http_referrer": "$http_referer", ' - '"http_user_agent": "$http_user_agent" } }'; - types_hash_max_size 2048; include /usr/local/nginx/conf/mime.types.default; default_type application/octet-stream; -access_log /mnt/logs/nginx.access.log logstash_json; +access_log /var/log/nginx/nginx.access.log; sendfile on; gzip on; diff --git a/conf/init/diffsworker/log/run b/conf/init/diffsworker/log/run new file mode 100755 index 000000000..066f7415a --- /dev/null +++ b/conf/init/diffsworker/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/diffsworker/ \ No newline at end of file diff --git a/conf/init/diffsworker.sh b/conf/init/diffsworker/run similarity index 51% rename from conf/init/diffsworker.sh rename to conf/init/diffsworker/run index 68d3c38b4..b40a1d034 100755 --- a/conf/init/diffsworker.sh +++ b/conf/init/diffsworker/run @@ -3,6 +3,6 @@ echo 'Starting diffs worker' cd / -venv/bin/python -m workers.diffsworker --log=/mnt/logs/diffsworker.log +venv/bin/python -m workers.diffsworker echo 'Diffs worker exited' \ No newline at end of file diff --git a/conf/init/dockerfilebuild/log/run b/conf/init/dockerfilebuild/log/run new file mode 100755 index 000000000..c971f6159 --- /dev/null +++ b/conf/init/dockerfilebuild/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/dockerfilebuild/ \ No newline at end of file diff --git a/conf/init/dockerfilebuild/run b/conf/init/dockerfilebuild/run new file mode 100755 index 000000000..b557a2823 --- /dev/null +++ b/conf/init/dockerfilebuild/run @@ -0,0 +1,6 @@ +#! /bin/bash + +sv start tutumdocker || exit 1 + +cd / +venv/bin/python -m workers.dockerfilebuild \ No newline at end of file diff --git a/conf/init/gunicorn/log/run b/conf/init/gunicorn/log/run new file mode 100755 index 000000000..106d6c4f8 --- /dev/null +++ b/conf/init/gunicorn/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/gunicorn/ \ No newline at end of file diff --git a/conf/init/gunicorn.sh b/conf/init/gunicorn/run similarity index 100% rename from conf/init/gunicorn.sh rename to conf/init/gunicorn/run diff --git a/conf/init/mklogsdir.sh b/conf/init/mklogsdir.sh deleted file mode 100755 index 4ca2880d0..000000000 --- a/conf/init/mklogsdir.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh - -echo 'Creating logs directory' -mkdir -p /mnt/logs \ No newline at end of file diff --git a/conf/init/nginx/log/run b/conf/init/nginx/log/run new file mode 100755 index 000000000..30476f6e6 --- /dev/null +++ b/conf/init/nginx/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/nginx/ \ No newline at end of file diff --git a/conf/init/nginx.sh b/conf/init/nginx/run similarity index 55% rename from conf/init/nginx.sh rename to conf/init/nginx/run index 6cd4b3430..e5cc1aaac 100755 --- a/conf/init/nginx.sh +++ b/conf/init/nginx/run @@ -5,10 +5,10 @@ echo 'Starting nginx' if [ -f /conf/stack/ssl.key ] then echo "Using HTTPS" - /usr/local/nginx/sbin/nginx -c /conf/nginx-enterprise.conf + /usr/local/nginx/sbin/nginx -c /conf/nginx.conf else echo "No SSL key provided, using HTTP" - /usr/local/nginx/sbin/nginx -c /conf/nginx-enterprise-nossl.conf + /usr/local/nginx/sbin/nginx -c /conf/nginx-nossl.conf fi echo 'Nginx exited' \ No newline at end of file diff --git a/conf/init/preplogsdir.sh b/conf/init/preplogsdir.sh new file mode 100755 index 000000000..8d9c67bfd --- /dev/null +++ b/conf/init/preplogsdir.sh @@ -0,0 +1,8 @@ +#! /bin/sh + +echo 'Linking config files to logs directory' +for svc in `ls /etc/service/` +do + mkdir -p /var/log/$svc + ln -s /svlogd_config /var/log/$svc/config +done diff --git a/conf/init/svlogd_config b/conf/init/svlogd_config new file mode 100644 index 000000000..de7984f15 --- /dev/null +++ b/conf/init/svlogd_config @@ -0,0 +1,2 @@ +s100000000 +t86400 diff --git a/conf/init/tutumdocker/log/run b/conf/init/tutumdocker/log/run new file mode 100755 index 000000000..dbabad38b --- /dev/null +++ b/conf/init/tutumdocker/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/tutumdocker/ \ No newline at end of file diff --git a/conf/init/tutumdocker/run b/conf/init/tutumdocker/run new file mode 100755 index 000000000..9221134b9 --- /dev/null +++ b/conf/init/tutumdocker/run @@ -0,0 +1,96 @@ +#!/bin/bash + +# First, make sure that cgroups are mounted correctly. +CGROUP=/sys/fs/cgroup + +[ -d $CGROUP ] || + mkdir $CGROUP + +mountpoint -q $CGROUP || + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use -privileged?" + exit 1 + } + +if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security +then + mount -t securityfs none /sys/kernel/security || { + echo "Could not mount /sys/kernel/security." + echo "AppArmor detection and -privileged mode might break." + } +fi + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) +do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS + + # The two following sections address a bug which manifests itself + # by a cryptic "lxc-start: no ns_cgroup option specified" when + # trying to start containers withina container. + # The bug seems to appear when the cgroup hierarchies are not + # mounted on the exact same directories in the host, and in the + # container. + + # Named, control-less cgroups are mounted with "-o name=foo" + # (and appear as such under /proc//cgroup) but are usually + # mounted on a directory named "foo" (without the "name=" prefix). + # Systemd and OpenRC (and possibly others) both create such a + # cgroup. To avoid the aforementioned bug, we symlink "foo" to + # "name=foo". This shouldn't have any adverse effect. + echo $SUBSYS | grep -q ^name= && { + NAME=$(echo $SUBSYS | sed s/^name=//) + ln -s $SUBSYS $CGROUP/$NAME + } + + # Likewise, on at least one system, it has been reported that + # systemd would mount the CPU and CPU accounting controllers + # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" + # but on a directory called "cpu,cpuacct" (note the inversion + # in the order of the groups). This tries to work around it. + [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +grep -q :devices: /proc/1/cgroup || + echo "WARNING: the 'devices' cgroup should be in its own hierarchy." +grep -qw devices /proc/1/cgroup || + echo "WARNING: it looks like the 'devices' cgroup is not mounted." + +# Now, close extraneous file descriptors. +pushd /proc/self/fd >/dev/null +for FD in * +do + case "$FD" in + # Keep stdin/stdout/stderr + [012]) + ;; + # Nuke everything else + *) + eval exec "$FD>&-" + ;; + esac +done +popd >/dev/null + + +# If a pidfile is still around (for example after a container restart), +# delete it so that docker can start. +rm -rf /var/run/docker.pid + +chmod 777 /var/lib/lxc +chmod 777 /var/lib/docker + + +# If we were given a PORT environment variable, start as a simple daemon; +# otherwise, spawn a shell as well +if [ "$PORT" ] +then + exec docker -d -H 0.0.0.0:$PORT +else + docker -d -D -e lxc 2>&1 +fi \ No newline at end of file diff --git a/conf/init/webhookworker/log/run b/conf/init/webhookworker/log/run new file mode 100755 index 000000000..6738f16f8 --- /dev/null +++ b/conf/init/webhookworker/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd -t /var/log/webhookworker/ \ No newline at end of file diff --git a/conf/init/webhookworker.sh b/conf/init/webhookworker/run similarity index 51% rename from conf/init/webhookworker.sh rename to conf/init/webhookworker/run index 7ab6340d8..04521552a 100755 --- a/conf/init/webhookworker.sh +++ b/conf/init/webhookworker/run @@ -3,6 +3,6 @@ echo 'Starting webhook worker' cd / -venv/bin/python -m workers.webhookworker --log=/mnt/logs/webhookworker.log +venv/bin/python -m workers.webhookworker echo 'Webhook worker exited' \ No newline at end of file diff --git a/conf/logging.conf b/conf/logging.conf index 2061a2375..4023e7743 100644 --- a/conf/logging.conf +++ b/conf/logging.conf @@ -1,38 +1,38 @@ [loggers] -keys=root, gunicorn.error, gunicorn.access +keys=root, gunicorn.error, gunicorn.access, application.profiler [handlers] -keys=error_file +keys=console [formatters] keys=generic [logger_application.profiler] level=DEBUG -handlers=error_file +handlers=console propagate=0 qualname=application.profiler [logger_root] level=DEBUG -handlers=error_file +handlers=console [logger_gunicorn.error] level=INFO -handlers=error_file +handlers=console propagate=1 qualname=gunicorn.error [logger_gunicorn.access] level=INFO -handlers=error_file +handlers=console propagate=0 qualname=gunicorn.access -[handler_error_file] -class=logging.FileHandler +[handler_console] +class=StreamHandler formatter=generic -args=('/mnt/logs/application.log',) +args=(sys.stdout, ) [formatter_generic] format=%(asctime)s [%(process)d] [%(levelname)s] [%(name)s] %(message)s diff --git a/conf/logging_local.conf b/conf/logging_local.conf deleted file mode 100644 index 4023e7743..000000000 --- a/conf/logging_local.conf +++ /dev/null @@ -1,39 +0,0 @@ -[loggers] -keys=root, gunicorn.error, gunicorn.access, application.profiler - -[handlers] -keys=console - -[formatters] -keys=generic - -[logger_application.profiler] -level=DEBUG -handlers=console -propagate=0 -qualname=application.profiler - -[logger_root] -level=DEBUG -handlers=console - -[logger_gunicorn.error] -level=INFO -handlers=console -propagate=1 -qualname=gunicorn.error - -[logger_gunicorn.access] -level=INFO -handlers=console -propagate=0 -qualname=gunicorn.access - -[handler_console] -class=StreamHandler -formatter=generic -args=(sys.stdout, ) - -[formatter_generic] -format=%(asctime)s [%(process)d] [%(levelname)s] [%(name)s] %(message)s -class=logging.Formatter diff --git a/conf/logrotate/quay-logrotate b/conf/logrotate/quay-logrotate deleted file mode 100644 index 1a6678639..000000000 --- a/conf/logrotate/quay-logrotate +++ /dev/null @@ -1,41 +0,0 @@ -/mnt/logs/nginx.access.log { - daily - rotate 7 - compress - delaycompress - missingok - notifempty - create 644 root root - - postrotate - kill -USR1 `cat /mnt/logs/nginx.pid` - endscript -} - -/mnt/logs/nginx.error.log { - daily - rotate 7 - compress - delaycompress - missingok - notifempty - create 644 root root - - postrotate - kill -USR1 `cat /mnt/logs/nginx.pid` - endscript -} - -/mnt/logs/application.log { - daily - rotate 7 - compress - delaycompress - missingok - notifempty - create 644 ubuntu ubuntu - - postrotate - kill -USR1 `cat /mnt/logs/gunicorn.pid` - endscript -} \ No newline at end of file diff --git a/conf/nginx-enterprise-nossl.conf b/conf/nginx-nossl.conf similarity index 100% rename from conf/nginx-enterprise-nossl.conf rename to conf/nginx-nossl.conf diff --git a/conf/nginx-enterprise.conf b/conf/nginx.conf similarity index 100% rename from conf/nginx-enterprise.conf rename to conf/nginx.conf diff --git a/conf/root-base.conf b/conf/root-base.conf index b4b9beb90..31c32d25d 100644 --- a/conf/root-base.conf +++ b/conf/root-base.conf @@ -1,5 +1,5 @@ pid /tmp/nginx.pid; -error_log /mnt/logs/nginx.error.log; +error_log /var/log/nginx/nginx.error.log; events { worker_connections 1024; diff --git a/conf/server-base.conf b/conf/server-base.conf index f37d83bba..6aeaa689e 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -1,7 +1,10 @@ client_max_body_size 8G; -client_body_temp_path /mnt/logs/client_body 1 2; +client_body_temp_path /var/log/nginx/client_body 1 2; server_name _; +set_real_ip_from 172.17.0.0/16; +real_ip_header X-Forwarded-For; + keepalive_timeout 5; if ($args ~ "_escaped_fragment_") { @@ -20,5 +23,5 @@ location / { proxy_pass http://app_server; proxy_read_timeout 2000; - proxy_temp_path /mnt/logs/proxy_temp 1 2; + proxy_temp_path /var/log/nginx/proxy_temp 1 2; } \ No newline at end of file diff --git a/config.py b/config.py index d5fc126cb..54650e566 100644 --- a/config.py +++ b/config.py @@ -73,7 +73,7 @@ class DefaultConfig(object): STORAGE_PATH = 'test/data/registry' # Build logs - BUILDLOGS = BuildLogs('logs.quay.io') # Change me + BUILDLOGS_OPTIONS = ['logs.quay.io'] # Real-time user events USER_EVENTS = UserEventBuilder('logs.quay.io') diff --git a/data/buildlogs.py b/data/buildlogs.py index 43723e211..c6acfd41b 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -1,10 +1,12 @@ import redis import json +from util.dynamic import import_class + class BuildStatusRetrievalError(Exception): pass -class BuildLogs(object): +class RedisBuildLogs(object): ERROR = 'error' COMMAND = 'command' PHASE = 'phase' @@ -70,3 +72,37 @@ class BuildLogs(object): raise BuildStatusRetrievalError('Cannot retrieve build status') return json.loads(fetched) if fetched else None + + def check_health(self): + try: + return self._redis.ping() == True + except redis.ConnectionError: + return False + + +class BuildLogs(object): + def __init__(self, app=None): + self.app = app + if app is not None: + self.state = self.init_app(app) + else: + self.state = None + + def init_app(self, app): + buildlogs_options = app.config.get('BUILDLOGS_OPTIONS', []) + buildlogs_import = app.config.get('BUILDLOGS_MODULE_AND_CLASS', None) + + if buildlogs_import is None: + klass = RedisBuildLogs + else: + klass = import_class(buildlogs_import[0], buildlogs_import[1]) + + buildlogs = klass(*buildlogs_options) + + # register extension with app + app.extensions = getattr(app, 'extensions', {}) + app.extensions['buildlogs'] = buildlogs + return buildlogs + + def __getattr__(self, name): + return getattr(self.state, name, None) \ No newline at end of file diff --git a/data/model/legacy.py b/data/model/legacy.py index 1c207cb79..bff044d5b 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1647,3 +1647,11 @@ def delete_user(user): user.delete_instance(recursive=True, delete_nullable=True) # TODO: also delete any repository data associated + +def check_health(): + # We will connect to the db, check that it contains some log entry kinds + try: + found_count = LogEntryKind.select().count() + return found_count > 0 + except: + return False diff --git a/endpoints/api/build.py b/endpoints/api/build.py index c9bd2cf3a..0255f1c60 100644 --- a/endpoints/api/build.py +++ b/endpoints/api/build.py @@ -3,7 +3,7 @@ import json from flask import request -from app import app, userfiles as user_files +from app import app, userfiles as user_files, build_logs from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, ApiResource, internal_only, format_date, api, Unauthorized, NotFound) @@ -17,7 +17,6 @@ from util.names import parse_robot_username logger = logging.getLogger(__name__) -build_logs = app.config['BUILDLOGS'] def get_trigger_config(trigger): diff --git a/endpoints/registry.py b/endpoints/registry.py index d701fd140..6c9800f5c 100644 --- a/endpoints/registry.py +++ b/endpoints/registry.py @@ -239,7 +239,7 @@ def put_image_checksum(namespace, repository, image_id): abort(400, "Missing checksum for image %(image_id)s", issue='missing-checksum', image_id=image_id) if not session.get('checksum'): - abort(400, 'Checksum not found in Cookie for image %(imaage_id)s', + abort(400, 'Checksum not found in Cookie for image %(image_id)s', issue='missing-checksum-cookie', image_id=image_id) profile.debug('Looking up repo image') diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 8114278d4..8a844a13c 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -315,8 +315,8 @@ class GithubBuildTrigger(BuildTrigger): def handle_trigger_request(self, request, auth_token, config): payload = request.get_json() - if not payload: - raise SkipRequestException() + if not payload or not 'head_commit' in payload: + raise SkipRequestException() if 'zen' in payload: raise ValidationRequestException() diff --git a/endpoints/web.py b/endpoints/web.py index 7bc539906..7cd95f1a1 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -2,13 +2,13 @@ import logging import os from flask import (abort, redirect, request, url_for, make_response, Response, - Blueprint, send_from_directory) + Blueprint, send_from_directory, jsonify) from flask.ext.login import current_user from urlparse import urlparse from data import model from data.model.oauth import DatabaseAuthorizationProvider -from app import app, billing as stripe +from app import app, billing as stripe, build_logs from auth.auth import require_session_login from auth.permissions import AdministerOrganizationPermission from util.invoice import renderInvoiceToPdf @@ -146,7 +146,16 @@ def v1(): @web.route('/status', methods=['GET']) @no_cache def status(): - return make_response('Healthy') + db_healthy = model.check_health() + buildlogs_healthy = build_logs.check_health() + + response = jsonify({ + 'db_healthy': db_healthy, + 'buildlogs_healthy': buildlogs_healthy, + }) + response.status_code = 200 if db_healthy and buildlogs_healthy else 503 + + return response @web.route('/tos', methods=['GET']) diff --git a/initdb.py b/initdb.py index 2570b7ca9..1aaa0ec1a 100644 --- a/initdb.py +++ b/initdb.py @@ -344,11 +344,6 @@ def populate_database(): 'docker_tags': ['latest'], 'build_subdir': '', } - build = model.create_repository_build(building, token, job_config, - '701dcc3724fb4f2ea6c31400528343cd', - 'build-name', trigger) - build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef' - build.save() build2 = model.create_repository_build(building, token, job_config, '68daeebd-a5b9-457f-80a0-4363b882f8ea', @@ -362,6 +357,12 @@ def populate_database(): build3.uuid = 'deadduck-dead-duck-dead-duckdeadduck' build3.save() + build = model.create_repository_build(building, token, job_config, + '701dcc3724fb4f2ea6c31400528343cd', + 'build-name', trigger) + build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef' + build.save() + org = model.create_organization('buynlarge', 'quay@devtable.com', new_user_1) org.stripe_id = TEST_STRIPE_ID diff --git a/screenshots/screenshots.js b/screenshots/screenshots.js index fbef4ac01..d7a94c3d4 100644 --- a/screenshots/screenshots.js +++ b/screenshots/screenshots.js @@ -1,4 +1,4 @@ -var width = 1024; +var width = 1060; var height = 768; var casper = require('casper').create({ @@ -76,11 +76,14 @@ casper.then(function() { this.capture(outputDir + 'repo-view.png'); }); -casper.then(function() { - this.log('Generating repository changes screenshot.'); + +casper.thenClick('a[data-image="c3d710edbd3b"]', function() { + this.waitForText('And 3048 more...', function() { + this.capture(outputDir + 'image-view.png'); + }); }); -casper.thenClick('#current-image dd a', function() { +casper.thenClick('.image-link', function() { this.waitForSelector('.result-count', function() { this.capture(outputDir + 'repo-changes.png', { top: 0, @@ -89,7 +92,8 @@ casper.thenClick('#current-image dd a', function() { height: height }); }); -}) +}); + casper.then(function() { this.log('Generating repository admin screenshot.'); diff --git a/static/img/build-history.png b/static/img/build-history.png index 794d58cb0..e53eb0301 100644 Binary files a/static/img/build-history.png and b/static/img/build-history.png differ diff --git a/static/img/org-admin.png b/static/img/org-admin.png index 831d21a22..f7bb9394f 100644 Binary files a/static/img/org-admin.png and b/static/img/org-admin.png differ diff --git a/static/img/org-logs.png b/static/img/org-logs.png index d9dd7ace4..9f6a14a22 100644 Binary files a/static/img/org-logs.png and b/static/img/org-logs.png differ diff --git a/static/img/org-repo-admin.png b/static/img/org-repo-admin.png index e41c798ad..466633e97 100644 Binary files a/static/img/org-repo-admin.png and b/static/img/org-repo-admin.png differ diff --git a/static/img/org-repo-list.png b/static/img/org-repo-list.png index 0198cad9e..dac3fd1d5 100644 Binary files a/static/img/org-repo-list.png and b/static/img/org-repo-list.png differ diff --git a/static/img/org-teams.png b/static/img/org-teams.png index 975b0f1af..da847f59f 100644 Binary files a/static/img/org-teams.png and b/static/img/org-teams.png differ diff --git a/static/img/repo-admin.png b/static/img/repo-admin.png index 891cdfc37..91903e1cf 100644 Binary files a/static/img/repo-admin.png and b/static/img/repo-admin.png differ diff --git a/static/img/repo-changes.png b/static/img/repo-changes.png index bb1ad0e43..6565ef21d 100644 Binary files a/static/img/repo-changes.png and b/static/img/repo-changes.png differ diff --git a/static/img/repo-view.png b/static/img/repo-view.png index aacbc65c3..7e52dd66a 100644 Binary files a/static/img/repo-view.png and b/static/img/repo-view.png differ diff --git a/static/img/user-home.png b/static/img/user-home.png index d42afc9c1..facc8a23d 100644 Binary files a/static/img/user-home.png and b/static/img/user-home.png differ diff --git a/static/js/app.js b/static/js/app.js index ada8cc4b1..4a7e68433 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -1,5 +1,5 @@ var TEAM_PATTERN = '^[a-zA-Z][a-zA-Z0-9]+$'; -var ROBOT_PATTERN = '^[a-zA-Z][a-zA-Z0-9]+$'; +var ROBOT_PATTERN = '^[a-zA-Z][a-zA-Z0-9]{3,29}$'; function getRestUrl(args) { var url = ''; @@ -61,7 +61,7 @@ function getFirstTextLine(commentString) { function createRobotAccount(ApiService, is_org, orgname, name, callback) { ApiService.createRobot(is_org ? orgname : null, null, {'robot_shortname': name}).then(callback, function(resp) { bootbox.dialog({ - "message": resp.data ? resp.data : 'The robot account could not be created', + "message": resp.data ? resp.data['message'] : 'The robot account could not be created', "title": "Cannot create robot account", "buttons": { "close": { @@ -84,7 +84,7 @@ function createOrganizationTeam(ApiService, orgname, teamname, callback) { 'teamname': teamname }; - ApiService.updateOrganizationTeam(data, params).then(callback, function() { + ApiService.updateOrganizationTeam(data, params).then(callback, function(resp) { bootbox.dialog({ "message": resp.data ? resp.data : 'The team could not be created', "title": "Cannot create team", @@ -3829,6 +3829,8 @@ quayApp.directive('setupTriggerDialog', function () { var modalSetup = false; $scope.show = function() { + if (!$scope.trigger || !$scope.repository) { return; } + $scope.activating = false; $scope.pullEntity = null; $scope.publicPull = true; @@ -3838,7 +3840,7 @@ quayApp.directive('setupTriggerDialog', function () { if (!modalSetup) { $('#setupTriggerModal').on('hidden.bs.modal', function () { - if ($scope.trigger['is_active']) { return; } + if (!$scope.trigger || $scope.trigger['is_active']) { return; } $scope.$apply(function() { $scope.cancelSetupTrigger(); diff --git a/static/js/controllers.js b/static/js/controllers.js index a39bcf7d5..d6e57a180 100644 --- a/static/js/controllers.js +++ b/static/js/controllers.js @@ -4,11 +4,16 @@ $.fn.clipboardCopy = function() { clip.on('complete', function() { // Resets the animation. var elem = $('#clipboardCopied')[0]; + if (!elem) { + return; + } + elem.style.display = 'none'; elem.classList.remove('animated'); // Show the notification. setTimeout(function() { + if (!elem) { return; } elem.style.display = 'inline-block'; elem.classList.add('animated'); }, 10); @@ -1071,7 +1076,6 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope $scope.currentParentEntry = null; $scope.currentBuild = build; - $scope.currentBuildIndex = index; if (opt_updateURL) { if (build) { @@ -1149,8 +1153,18 @@ function RepoBuildCtrl($scope, Restangular, ApiService, $routeParams, $rootScope ApiService.getRepoBuildStatus(null, params, true).then(function(resp) { // Note: We use extend here rather than replacing as Angular is depending on the // root build object to remain the same object. - $.extend(true, $scope.builds[$scope.currentBuildIndex], resp); - var currentBuild = $scope.builds[$scope.currentBuildIndex]; + var matchingBuilds = $.grep($scope.builds, function(elem) { + return elem['id'] == resp['id'] + }); + + var currentBuild = matchingBuilds.length > 0 ? matchingBuilds[0] : null; + if (currentBuild) { + currentBuild = $.extend(true, currentBuild, resp); + } else { + currentBuild = resp; + $scope.builds.push(currentBuild); + } + checkPollTimer(); // Load the updated logs for the build. @@ -1239,12 +1253,12 @@ function RepoAdminCtrl($scope, Restangular, ApiService, KeyService, $routeParams $scope.getBadgeFormat = function(format, repo) { if (!repo) { return; } - var imageUrl = Config.getUrl('/' + namespace + '/' + name + '/status'); + var imageUrl = Config.getUrl('/repository/' + namespace + '/' + name + '/status'); if (!$scope.repo.is_public) { imageUrl += '?token=' + $scope.repo.status_token; } - var linkUrl = Config.getUrl('/' + namespace + '/' + name); + var linkUrl = Config.getUrl('/repository/' + namespace + '/' + name); switch (format) { case 'svg': @@ -1642,12 +1656,14 @@ function UserAdminCtrl($scope, $timeout, $location, ApiService, PlanService, Use $scope.cuser = jQuery.extend({}, user); - for (var i = 0; i < $scope.cuser.logins.length; i++) { - if ($scope.cuser.logins[i].service == 'github') { - var githubId = $scope.cuser.logins[i].service_identifier; - $http.get('https://api.github.com/user/' + githubId).success(function(resp) { - $scope.githubLogin = resp.login; - }); + if ($scope.cuser.logins) { + for (var i = 0; i < $scope.cuser.logins.length; i++) { + if ($scope.cuser.logins[i].service == 'github') { + var githubId = $scope.cuser.logins[i].service_identifier; + $http.get('https://api.github.com/user/' + githubId).success(function(resp) { + $scope.githubLogin = resp.login; + }); + } } } }); @@ -1940,7 +1956,7 @@ function NewRepoCtrl($scope, $location, $http, $timeout, UserService, ApiService $scope.githubClientId = KeyService.githubClientId; $scope.repo = { - 'is_public': 1, + 'is_public': 0, 'description': '', 'initialize': '' }; @@ -1959,9 +1975,6 @@ function NewRepoCtrl($scope, $location, $http, $timeout, UserService, ApiService // Determine whether private repositories are allowed for the namespace. checkPrivateAllowed(); - - // Default to private repos for organizations. - $scope.repo.is_public = isUserNamespace ? '1' : '0'; }); $scope.changeNamespace = function(namespace) { diff --git a/static/partials/new-repo.html b/static/partials/new-repo.html index e4f74c639..d2cd8941a 100644 --- a/static/partials/new-repo.html +++ b/static/partials/new-repo.html @@ -166,7 +166,8 @@
diff --git a/static/partials/view-repo.html b/static/partials/view-repo.html index b5d039974..eedeece40 100644 --- a/static/partials/view-repo.html +++ b/static/partials/view-repo.html @@ -177,7 +177,12 @@ - {{ image.id.substr(0, 12) }} + + + {{ image.id.substr(0, 12) }} + + @@ -199,7 +204,7 @@
- {{ currentImage.id }} + {{ currentImage.id }}
diff --git a/test/data/test.db b/test/data/test.db index 607f262d6..dc52ca234 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_usage.py b/test/test_api_usage.py index a1683b372..c53d46f01 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -68,6 +68,8 @@ CSRF_TOKEN_KEY = '_csrf_token' CSRF_TOKEN = '123csrfforme' class ApiTestCase(unittest.TestCase): + maxDiff = None + @staticmethod def _add_csrf(without_csrf): parts = urlparse(without_csrf) @@ -968,7 +970,7 @@ class TestRepoBuilds(ApiTestCase): params=dict(repository=ADMIN_ACCESS_USER + '/building')) assert len(json['builds']) > 0 - build = json['builds'][0] + build = json['builds'][-1] assert 'id' in build assert 'status' in build diff --git a/test/testconfig.py b/test/testconfig.py index d012af469..e03c2328f 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -24,7 +24,8 @@ class TestConfig(DefaultConfig): STORAGE_TYPE = 'FakeStorage' - BUILDLOGS = TestBuildLogs('logs.quay.io', 'devtable', 'building', - 'deadbeef-dead-beef-dead-beefdeadbeef') + BUILDLOGS_MODULE_AND_CLASS = ('test.testlogs', 'testlogs.TestBuildLogs') + BUILDLOGS_OPTIONS = ['logs.quay.io', 'devtable', 'building', + 'deadbeef-dead-beef-dead-beefdeadbeef'] USERFILES_TYPE = 'FakeUserfiles' diff --git a/test/testlogs.py b/test/testlogs.py index b8e399dad..fa3c2bec2 100644 --- a/test/testlogs.py +++ b/test/testlogs.py @@ -5,7 +5,7 @@ from loremipsum import get_sentence from functools import wraps from copy import deepcopy -from data.buildlogs import BuildLogs +from data.buildlogs import RedisBuildLogs logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ def maybe_advance_script(is_get_status=False): return inner_advance -class TestBuildLogs(BuildLogs): +class TestBuildLogs(RedisBuildLogs): COMMAND_TYPES = ['FROM', 'MAINTAINER', 'RUN', 'CMD', 'EXPOSE', 'ENV', 'ADD', 'ENTRYPOINT', 'VOLUME', 'USER', 'WORKDIR'] STATUS_TEMPLATE = { diff --git a/util/dynamic.py b/util/dynamic.py new file mode 100644 index 000000000..975a4d930 --- /dev/null +++ b/util/dynamic.py @@ -0,0 +1,7 @@ +def import_class(module_name, class_name): + """ Import a class given the specified module name and class name. """ + klass = __import__(module_name) + class_segments = class_name.split('.') + for segment in class_segments: + klass = getattr(klass, segment) + return klass diff --git a/util/exceptionlog.py b/util/exceptionlog.py index 462e3229b..b95ca6822 100644 --- a/util/exceptionlog.py +++ b/util/exceptionlog.py @@ -1,7 +1,15 @@ from raven.contrib.flask import Sentry as FlaskSentry +class FakeSentryClient(object): + def captureException(self, *args, **kwargs): + pass + + def user_context(self, *args, **kwargs): + pass + class FakeSentry(object): - pass + def __init__(self): + self.client = FakeSentryClient() class Sentry(object): def __init__(self, app=None): diff --git a/util/phantomjs-runner.js b/util/phantomjs-runner.js index 30b0439fa..fae6496e0 100644 --- a/util/phantomjs-runner.js +++ b/util/phantomjs-runner.js @@ -1,37 +1,55 @@ var system = require('system'); var url = system.args[1] || ''; +var count = 0; + if(url.length > 0) { var page = require('webpage').create(); page.open(url, function (status) { - if (status == 'success') { - var delay, checker = (function() { - var html = page.evaluate(function () { - var found = document.getElementsByTagName('html')[0].outerHTML || ''; - if (window.__isLoading && !window.__isLoading()) { - return found; - } - if (found.indexOf('404 Not Found') > 0) { - return found; - } - return null; - }); + try { + if (status == 'success') { + var delay; + var checker = (function() { + count++; - if (html) { - if (html.indexOf('404 Not Found') > 0) { + if (count > 100) { console.log('Not Found'); phantom.exit(); - return; + return null; } - clearTimeout(delay); - console.log(html); - phantom.exit(); - } - }); - delay = setInterval(checker, 100); - } else { + var html = page.evaluate(function () { + var found = document.getElementsByTagName('html')[0].outerHTML || ''; + if (window.__isLoading && !window.__isLoading()) { + return found; + } + if (found.indexOf('404 Not Found') > 0) { + return found; + } + return null; + }); + + if (html) { + if (html.indexOf('404 Not Found') > 0) { + console.log('Not Found'); + phantom.exit(); + return; + } + + clearTimeout(delay); + console.log(html); + phantom.exit(); + } + }); + delay = setInterval(checker, 100); + } else { + console.log('Not Found'); + phantom.exit(); + } + } catch (e) { console.log('Not Found'); phantom.exit(); } }); +} else { + phantom.exit(); } \ No newline at end of file diff --git a/util/seo.py b/util/seo.py index 42af53502..b75480661 100644 --- a/util/seo.py +++ b/util/seo.py @@ -3,22 +3,27 @@ import logging from bs4 import BeautifulSoup - logger = logging.getLogger(__name__) def render_snapshot(url): logger.info('Snapshotting url: %s' % url) + out_html = subprocess.check_output(['phantomjs', '--ignore-ssl-errors=yes', + '--disk-cache=yes', 'util/phantomjs-runner.js', url]) if not out_html or out_html.strip() == 'Not Found': return None # Remove script tags + logger.info('Removing script tags: %s' % url) + soup = BeautifulSoup(out_html.decode('utf8')) to_extract = soup.findAll('script') for item in to_extract: item.extract() + logger.info('Snapshotted url: %s' % url) + return str(soup) diff --git a/workers/README.md b/workers/README.md index e91246861..f65164d89 100644 --- a/workers/README.md +++ b/workers/README.md @@ -1,31 +1,35 @@ -to prepare a new build node host starting from a 14.04 base server: +to build and upload the builder to quay ``` -sudo apt-get update -sudo apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core lxc -``` - -check out the code, install the kernel, custom docker, nsexec, and reboot: - -``` -git clone https://bitbucket.org/yackob03/quay.git +curl -s https://get.docker.io/ubuntu/ | sudo sh +sudo apt-get update && sudo apt-get install -y git +git clone git clone https://bitbucket.org/yackob03/quay.git cd quay -sudo gdebi --n binary_dependencies/builder/nsexec_1.22ubuntu1trusty1_amd64.deb -sudo gdebi --n binary_dependencies/builder/lxc-docker-0.9.0_0.9.0-20140501212101-72572f0-dirty_amd64.deb -sudo usermod -v 100000-200000 -w 100000-200000 root -sudo chmod +x /var/lib/lxc -sudo chmod +x /var/lib/docker -cd ~ -git clone https://bitbucket.org/yackob03/quayconfig.git -ln -s ../../quayconfig/production/ quay/conf/stack +rm Dockerfile +ln -s Dockerfile.buildworker Dockerfile +sudo docker build -t quay.io/quay/builder . +sudo docker push quay.io/quay/builder +``` + +to run the code from a fresh 14.04 server: + +``` +sudo apt-get update && sudo apt-get install -y git lxc linux-image-extra-`uname -r` +curl -s https://get.docker.io/ubuntu/ | sudo sh +git clone https://github.com/DevTable/gantryd.git +cd gantryd +cat requirements.system | xargs sudo apt-get install -y +virtualenv --distribute venv +venv/bin/pip install -r requirements.txt +sudo docker login -p 9Y1PX7D3IE4KPSGCIALH17EM5V3ZTMP8CNNHJNXAQ2NJGAS48BDH8J1PUOZ869ML -u 'quay+deploy' -e notused quay.io ``` start the worker ``` -cd quay -virtualenv --distribute venv -source venv/bin/activate -pip install -r requirements.txt -sudo venv/bin/python -m workers.dockerfilebuild -D +cd ~ +git clone https://bitbucket.org/yackob03/quayconfig.git +sudo docker pull quay.io/quay/builder +cd ~/gantryd +sudo venv/bin/python gantry.py ../quayconfig/production/gantry.json update builder ``` diff --git a/workers/diffsworker.py b/workers/diffsworker.py index c5d9b2b5a..85b615cbe 100644 --- a/workers/diffsworker.py +++ b/workers/diffsworker.py @@ -33,16 +33,7 @@ class DiffsWorker(Worker): return True -parser = argparse.ArgumentParser(description='Worker daemon to compute diffs') -parser.add_argument('--log', help='Specify the log file for the worker as a daemon.') -args = parser.parse_args() - -if args.log is not None: - handler = logging.FileHandler(args.log) -else: - handler = logging.StreamHandler() -handler.setFormatter(formatter) -root_logger.addHandler(handler) +logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) worker = DiffsWorker(image_diff_queue) -worker.start() \ No newline at end of file +worker.start() diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index b6a098fe5..dbd9ab3a4 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -21,7 +21,7 @@ from collections import defaultdict from data.queue import dockerfile_build_queue from data import model from workers.worker import Worker, WorkerUnhealthyException, JobException -from app import app, userfiles as user_files +from app import userfiles as user_files, build_logs, sentry from util.safetar import safe_extractall from util.dockerfileparse import parse_dockerfile, ParsedDockerfile, serialize_dockerfile @@ -34,8 +34,6 @@ formatter = logging.Formatter(FORMAT) logger = logging.getLogger(__name__) -build_logs = app.config['BUILDLOGS'] - TIMEOUT_PERIOD_MINUTES = 20 CACHE_EXPIRATION_PERIOD_HOURS = 24 NO_TAGS = [':'] @@ -143,6 +141,7 @@ class DockerfileBuildContext(object): self.__cleanup_images() self.__prune_cache() except APIError: + sentry.client.captureException() message = 'Docker installation is no longer healthy.' logger.exception(message) raise WorkerUnhealthyException(message) @@ -452,6 +451,9 @@ class DockerfileBuildWorker(Worker): def process_queue_item(self, job_details): self._timeout.clear() + # Make sure we have more information for debugging problems + sentry.client.user_context(job_details) + repository_build = model.get_repository_build(job_details['namespace'], job_details['repository'], job_details['build_uuid']) @@ -542,6 +544,7 @@ class DockerfileBuildWorker(Worker): raise exc except Exception as exc: + sentry.client.captureException() log_appender('error', build_logs.PHASE) logger.exception('Exception when processing request.') repository_build.phase = 'error' @@ -552,27 +555,12 @@ class DockerfileBuildWorker(Worker): desc = 'Worker daemon to monitor dockerfile build' parser = argparse.ArgumentParser(description=desc) -parser.add_argument('-D', action='store_true', default=False, - help='Run the worker in daemon mode.') -parser.add_argument('--log', default='dockerfilebuild.log', - help='Specify the log file for the worker as a daemon.') parser.add_argument('--cachegb', default=20, type=float, help='Maximum cache size in gigabytes.') args = parser.parse_args() +logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) worker = DockerfileBuildWorker(args.cachegb, dockerfile_build_queue, reservation_seconds=RESERVATION_TIME) - -if args.D: - handler = logging.FileHandler(args.log) - handler.setFormatter(formatter) - root_logger.addHandler(handler) - with daemon.DaemonContext(files_preserve=[handler.stream]): - worker.start() - -else: - handler = logging.StreamHandler() - handler.setFormatter(formatter) - root_logger.addHandler(handler) - worker.start() +worker.start(start_status_server_port=8000) diff --git a/workers/webhookworker.py b/workers/webhookworker.py index f3d193ee9..2b785acb6 100644 --- a/workers/webhookworker.py +++ b/workers/webhookworker.py @@ -34,17 +34,7 @@ class WebhookWorker(Worker): return True - -parser = argparse.ArgumentParser(description='Worker daemon to send webhooks') -parser.add_argument('--log', help='Specify the log file for the worker as a daemon.') -args = parser.parse_args() - -if args.log is not None: - handler = logging.FileHandler(args.log) -else: - handler = logging.StreamHandler() -handler.setFormatter(formatter) -root_logger.addHandler(handler) +logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) worker = WebhookWorker(webhook_queue, poll_period_seconds=15, reservation_seconds=3600) diff --git a/workers/worker.py b/workers/worker.py index 2186087ba..112c4f6bc 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -1,11 +1,16 @@ import logging import json import signal +import sys from threading import Event from apscheduler.scheduler import Scheduler from datetime import datetime, timedelta +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from threading import Thread +from time import sleep +from data.model import db logger = logging.getLogger(__name__) @@ -23,6 +28,36 @@ class WorkerUnhealthyException(Exception): pass +class WorkerStatusServer(HTTPServer): + def __init__(self, worker, *args, **kwargs): + HTTPServer.__init__(self, *args, **kwargs) + self.worker = worker + + +class WorkerStatusHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == '/status': + # Return the worker status + code = 200 if self.server.worker.is_healthy() else 503 + self.send_response(code) + elif self.path == '/terminate': + # Return whether it is safe to terminate the worker process + code = 200 if self.server.worker.is_terminated() else 503 + self.send_response(code) + else: + self.send_error(404) + + def do_POST(self): + if self.path == '/terminate': + try: + self.server.worker.join() + self.send_response(200) + except: + self.send_response(500) + else: + self.send_error(404) + + class Worker(object): def __init__(self, queue, poll_period_seconds=30, reservation_seconds=300, watchdog_period_seconds=60): @@ -31,6 +66,7 @@ class Worker(object): self._reservation_seconds = reservation_seconds self._watchdog_period_seconds = watchdog_period_seconds self._stop = Event() + self._terminated = Event() self._queue = queue self.current_queue_item = None @@ -42,6 +78,17 @@ class Worker(object): """ Function that gets run once every watchdog_period_seconds. """ pass + def _close_db_handle(self): + if not db.is_closed(): + logger.debug('Disconnecting from database.') + db.close() + + def is_healthy(self): + return not self._stop.is_set() + + def is_terminated(self): + return self._terminated.is_set() + def extend_processing(self, seconds_from_now): if self.current_queue_item is not None: self._queue.extend_processing(self.current_queue_item, seconds_from_now) @@ -51,7 +98,7 @@ class Worker(object): self.current_queue_item = self._queue.get() while self.current_queue_item: - logger.debug('Queue gave us some work: %s' % self.current_queue_item.body) + logger.debug('Queue gave us some work: %s', self.current_queue_item.body) job_details = json.loads(self.current_queue_item.body) @@ -68,13 +115,24 @@ class Worker(object): finally: self.current_queue_item = None + # Close the db handle periodically + self._close_db_handle() + if not self._stop.is_set(): self.current_queue_item = self._queue.get(processing_time=self._reservation_seconds) if not self._stop.is_set(): logger.debug('No more work.') - def start(self): + def start(self, start_status_server_port=None): + if start_status_server_port is not None: + # Start a status server on a thread + server_address = ('', start_status_server_port) + httpd = WorkerStatusServer(self, server_address, WorkerStatusHandler) + server_thread = Thread(target=httpd.serve_forever) + server_thread.daemon = True + server_thread.start() + logger.debug("Scheduling worker.") soon = datetime.now() + timedelta(seconds=.001) @@ -84,8 +142,8 @@ class Worker(object): start_date=soon) self._sched.add_interval_job(self.watchdog, seconds=self._watchdog_period_seconds) - signal.signal(signal.SIGTERM, self.join) - signal.signal(signal.SIGINT, self.join) + signal.signal(signal.SIGTERM, self.terminate) + signal.signal(signal.SIGINT, self.terminate) while not self._stop.wait(1): pass @@ -94,11 +152,25 @@ class Worker(object): self._sched.shutdown() logger.debug('Finished.') - def join(self, signal_num=None, stack_frame=None): - logger.debug('Shutting down worker gracefully.') - self._stop.set() + self._terminated.set() - # Give back the retry that we took for this queue item so that if it were down to zero - # retries it will still be picked up by another worker - if self.current_queue_item is not None: - self._queue.incomplete(self.current_queue_item, restore_retry=True) + # Wait forever if we're running a server + while start_status_server_port is not None: + sleep(60) + + def terminate(self, signal_num=None, stack_frame=None, graceful=False): + if self._terminated.is_set(): + sys.exit(1) + + else: + logger.debug('Shutting down worker.') + self._stop.set() + + if not graceful: + # Give back the retry that we took for this queue item so that if it were down to zero + # retries it will still be picked up by another worker + if self.current_queue_item is not None: + self._queue.incomplete(self.current_queue_item, restore_retry=True) + + def join(self): + self.terminate(graceful=True)