diff --git a/Dockerfile b/Dockerfile index bca6b339c..6a036bdad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,17 @@ -FROM ubuntu:13.10 +FROM phusion/baseimage:0.9.9 ENV DEBIAN_FRONTEND noninteractive +ENV HOME /root + RUN apt-get update -RUN apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core +RUN apt-get install -y git python-virtualenv python-dev phantomjs libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev libevent-dev gdebi-core g++ ADD binary_dependencies binary_dependencies RUN gdebi --n binary_dependencies/*.deb -ADD requirements.enterprise requirements.txt +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ADD requirements.txt requirements.txt RUN virtualenv --distribute venv RUN venv/bin/pip install -r requirements.txt @@ -28,10 +32,15 @@ ADD workers workers ADD app.py app.py ADD application.py application.py ADD config.py config.py -ADD run.sh run.sh + +ADD conf/init/mklogsdir.sh /etc/my_init.d/ +ADD conf/init/gunicorn.sh /etc/service/gunicorn/run +ADD conf/init/nginx.sh /etc/service/nginx/run +ADD conf/init/diffsworker.sh /etc/service/diffsworker/run +ADD conf/init/webhookworker.sh /etc/service/webhookworker/run VOLUME ["/conf/stack", "/mnt/logs"] EXPOSE 443 80 -ENTRYPOINT ["./run.sh"] \ No newline at end of file +CMD ["/sbin/my_init"] \ No newline at end of file diff --git a/README.md b/README.md index fdacaba51..eb7c477d2 100644 --- a/README.md +++ b/README.md @@ -15,30 +15,13 @@ sudo mkdir -p /mnt/logs/ sudo docker run -d -p 80:80 -p 443:443 -v /mnt/logs:/mnt/logs -v `pwd`/quayconfig/production:/conf/stack quay.io/quay/quay ``` -start the log shipper: +start the log shipper (DEPRECATED): ``` sudo docker pull quay.io/quay/logstash sudo docker run -d -e REDIS_PORT_6379_TCP_ADDR=logs.quay.io -v /mnt/logs:/mnt/logs quay.io/quay/logstash quay.conf ``` -start the workers (FIXME): - -``` -STACK=prod python -m workers.diffsworker -D -STACK=prod python -m workers.webhookworker -D -``` - -bouncing the servers (FIXME): - -``` -sudo kill -HUP `cat /mnt/logs/nginx.pid` -kill -HUP `cat /mnt/logs/gunicorn.pid` - -kill -restart daemons -``` - running the tests: ``` diff --git a/binary_dependencies/nginx_1.4.2-nobuffer-2_amd64.deb b/binary_dependencies/nginx_1.4.2-nobuffer-2_amd64.deb new file mode 100644 index 000000000..dfc530e7f Binary files /dev/null and b/binary_dependencies/nginx_1.4.2-nobuffer-2_amd64.deb differ diff --git a/conf/gunicorn_config.py b/conf/gunicorn_config.py index c7332fe96..162c617b9 100644 --- a/conf/gunicorn_config.py +++ b/conf/gunicorn_config.py @@ -2,7 +2,6 @@ bind = 'unix:/tmp/gunicorn.sock' workers = 8 worker_class = 'gevent' timeout = 2000 -daemon = True pidfile = '/mnt/logs/gunicorn.pid' errorlog = '/mnt/logs/application.log' loglevel = 'debug' diff --git a/conf/init/diffsworker.sh b/conf/init/diffsworker.sh new file mode 100755 index 000000000..68d3c38b4 --- /dev/null +++ b/conf/init/diffsworker.sh @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting diffs worker' + +cd / +venv/bin/python -m workers.diffsworker --log=/mnt/logs/diffsworker.log + +echo 'Diffs worker exited' \ No newline at end of file diff --git a/conf/init/gunicorn.sh b/conf/init/gunicorn.sh new file mode 100755 index 000000000..a61e7c651 --- /dev/null +++ b/conf/init/gunicorn.sh @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting gunicon' + +cd / +venv/bin/gunicorn -c conf/gunicorn_config.py application:application + +echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/init/mklogsdir.sh b/conf/init/mklogsdir.sh new file mode 100755 index 000000000..4ca2880d0 --- /dev/null +++ b/conf/init/mklogsdir.sh @@ -0,0 +1,4 @@ +#! /bin/sh + +echo 'Creating logs directory' +mkdir -p /mnt/logs \ No newline at end of file diff --git a/conf/init/nginx.sh b/conf/init/nginx.sh new file mode 100755 index 000000000..6cd4b3430 --- /dev/null +++ b/conf/init/nginx.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +echo 'Starting nginx' + +if [ -f /conf/stack/ssl.key ] +then + echo "Using HTTPS" + /usr/local/nginx/sbin/nginx -c /conf/nginx-enterprise.conf +else + echo "No SSL key provided, using HTTP" + /usr/local/nginx/sbin/nginx -c /conf/nginx-enterprise-nossl.conf +fi + +echo 'Nginx exited' \ No newline at end of file diff --git a/conf/init/webhookworker.sh b/conf/init/webhookworker.sh new file mode 100755 index 000000000..7ab6340d8 --- /dev/null +++ b/conf/init/webhookworker.sh @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting webhook worker' + +cd / +venv/bin/python -m workers.webhookworker --log=/mnt/logs/webhookworker.log + +echo 'Webhook worker exited' \ No newline at end of file diff --git a/conf/nginx-enterprise-nossl.conf b/conf/nginx-enterprise-nossl.conf index 6561e4c10..73a9c7605 100644 --- a/conf/nginx-enterprise-nossl.conf +++ b/conf/nginx-enterprise-nossl.conf @@ -4,6 +4,8 @@ worker_processes 2; user root nogroup; +daemon off; + http { include http-base.conf; diff --git a/conf/nginx-enterprise.conf b/conf/nginx-enterprise.conf index b704dc2db..43c21b6ca 100644 --- a/conf/nginx-enterprise.conf +++ b/conf/nginx-enterprise.conf @@ -4,6 +4,8 @@ worker_processes 2; user root nogroup; +daemon off; + http { include http-base.conf; diff --git a/requirements.txt b/requirements.txt index 3df9f03e0..06f341226 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,6 @@ argparse==1.2.1 beautifulsoup4==4.3.2 blinker==1.3 boto==2.27.0 -distribute==0.6.34 git+https://github.com/DevTable/docker-py.git ecdsa==0.11 gevent==1.0 diff --git a/test/data/test.db b/test/data/test.db index 88b7213c5..6471bc3e4 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/workers/diffsworker.py b/workers/diffsworker.py index 1d6ee56af..c5d9b2b5a 100644 --- a/workers/diffsworker.py +++ b/workers/diffsworker.py @@ -1,5 +1,4 @@ import logging -import daemon import argparse from data.queue import image_diff_queue @@ -34,26 +33,16 @@ class DiffsWorker(Worker): return True - parser = argparse.ArgumentParser(description='Worker daemon to compute diffs') -parser.add_argument('-D', action='store_true', default=False, - help='Run the worker in daemon mode.') -parser.add_argument('--log', default='diffsworker.log', - help='Specify the log file for the worker as a daemon.') +parser.add_argument('--log', help='Specify the log file for the worker as a daemon.') args = parser.parse_args() - -worker = DiffsWorker(image_diff_queue) - -if args.D: +if args.log is not None: handler = logging.FileHandler(args.log) - handler.setFormatter(formatter) - root_logger.addHandler(handler) - with daemon.DaemonContext(files_preserve=[handler.stream]): - worker.start() - else: handler = logging.StreamHandler() - handler.setFormatter(formatter) - root_logger.addHandler(handler) - worker.start() \ No newline at end of file +handler.setFormatter(formatter) +root_logger.addHandler(handler) + +worker = DiffsWorker(image_diff_queue) +worker.start() \ No newline at end of file diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 21d4c5cab..4d7b0e306 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -461,7 +461,7 @@ parser.add_argument('--log', default='dockerfilebuild.log', args = parser.parse_args() -worker = DockerfileBuildWorker(dockerfile_build_queue) +worker = DockerfileBuildWorker(dockerfile_build_queue, reservation_seconds=60*60) # 1 hour if args.D: handler = logging.FileHandler(args.log) diff --git a/workers/webhookworker.py b/workers/webhookworker.py index 575c11f41..f3d193ee9 100644 --- a/workers/webhookworker.py +++ b/workers/webhookworker.py @@ -1,5 +1,4 @@ import logging -import daemon import argparse import requests import json @@ -37,25 +36,16 @@ class WebhookWorker(Worker): parser = argparse.ArgumentParser(description='Worker daemon to send webhooks') -parser.add_argument('-D', action='store_true', default=False, - help='Run the worker in daemon mode.') -parser.add_argument('--log', default='webhooks.log', - help='Specify the log file for the worker as a daemon.') +parser.add_argument('--log', help='Specify the log file for the worker as a daemon.') args = parser.parse_args() +if args.log is not None: + handler = logging.FileHandler(args.log) +else: + handler = logging.StreamHandler() +handler.setFormatter(formatter) +root_logger.addHandler(handler) worker = WebhookWorker(webhook_queue, poll_period_seconds=15, reservation_seconds=3600) - -if args.D: - handler = logging.FileHandler(args.log) - handler.setFormatter(formatter) - root_logger.addHandler(handler) - with daemon.DaemonContext(files_preserve=[handler.stream]): - worker.start() - -else: - handler = logging.StreamHandler() - handler.setFormatter(formatter) - root_logger.addHandler(handler) - worker.start() \ No newline at end of file +worker.start() \ No newline at end of file diff --git a/workers/worker.py b/workers/worker.py index a525913a1..5b6d6785c 100644 --- a/workers/worker.py +++ b/workers/worker.py @@ -1,5 +1,6 @@ import logging import json +import signal from threading import Event from apscheduler.scheduler import Scheduler @@ -52,8 +53,16 @@ class Worker(object): self._sched.add_interval_job(self.poll_queue, seconds=self._poll_period_seconds) self._sched.add_interval_job(self.watchdog, seconds=self._watchdog_period_seconds) + signal.signal(signal.SIGTERM, self.join) + signal.signal(signal.SIGINT, self.join) + while not self._stop.wait(1): pass - def join(self): + logger.debug('Waiting for running tasks to complete.') + self._sched.shutdown() + logger.debug('Finished.') + + def join(self, signal_num=None, stack_frame=None): + logger.debug('Shutting down worker gracefully.') self._stop.set()