diff --git a/Dockerfile.buildworker b/Dockerfile.buildworker index 4b6f995b9..09c1c91b7 100644 --- a/Dockerfile.buildworker +++ b/Dockerfile.buildworker @@ -1,23 +1,30 @@ # vim:ft=dockerfile + +############################### +# BEGIN COMMON SECION +############################### + FROM phusion/baseimage:0.9.15 ENV DEBIAN_FRONTEND noninteractive ENV HOME /root # Install the dependencies. -RUN apt-get update # 20NOV2014 +RUN apt-get update # 11DEC2014 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev # Build the python dependencies ADD requirements.txt requirements.txt RUN virtualenv --distribute venv RUN venv/bin/pip install -r requirements.txt -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev +RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev -### End common section ### +############################### +# END COMMON SECION +############################### RUN apt-get install -y lxc aufs-tools diff --git a/Dockerfile.web b/Dockerfile.web index bd07d9999..d50256b2a 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,4 +1,9 @@ # vim:ft=dockerfile + +############################### +# BEGIN COMMON SECION +############################### + FROM phusion/baseimage:0.9.15 ENV DEBIAN_FRONTEND noninteractive @@ -8,16 +13,18 @@ ENV HOME /root RUN apt-get update # 11DEC2014 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands -RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev +RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev # Build the python dependencies ADD requirements.txt requirements.txt RUN virtualenv --distribute venv RUN venv/bin/pip install -r requirements.txt -RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev +RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev -### End common section ### +############################### +# END COMMON SECION +############################### # Remove SSH. RUN rm -rf /etc/service/sshd /etc/my_init.d/00_regen_ssh_host_keys.sh diff --git a/buildman/builder.py b/buildman/builder.py index df485f142..4e88d3ed7 100644 --- a/buildman/builder.py +++ b/buildman/builder.py @@ -49,8 +49,8 @@ def run_build_manager(): if os.environ.get('SSL_CONFIG'): logger.debug('Loading SSL cert and key') ssl_context = SSLContext() - ssl_context.load_cert_chain(os.environ.get('SSL_CONFIG') + '/ssl.cert', - os.environ.get('SSL_CONFIG') + '/ssl.key') + ssl_context.load_cert_chain(os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.cert'), + os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.key')) server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs, user_files, manager_klass, build_manager_config[1], public_ip) diff --git a/buildman/manager/basemanager.py b/buildman/manager/basemanager.py index fc9fd70cf..76e97e5ac 100644 --- a/buildman/manager/basemanager.py +++ b/buildman/manager/basemanager.py @@ -59,3 +59,9 @@ class BaseManager(object): automatically requeued. """ raise NotImplementedError + + def num_workers(self): + """ Returns the number of active build workers currently registered. This includes those + that are currently busy and awaiting more work. + """ + raise NotImplementedError diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py index 516464ff3..5a97c0955 100644 --- a/buildman/manager/enterprise.py +++ b/buildman/manager/enterprise.py @@ -74,3 +74,5 @@ class EnterpriseManager(BaseManager): if build_component in self.ready_components: self.ready_components.remove(build_component) + def num_workers(self): + return len(self.build_components) diff --git a/buildman/server.py b/buildman/server.py index 66f0010b6..576bb3a10 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -9,7 +9,7 @@ from aiowsgi import create_server as create_wsgi_server from flask import Flask from threading import Event from trollius.coroutines import From -from datetime import datetime, timedelta +from datetime import timedelta from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException from data.queue import WorkQueue @@ -140,7 +140,7 @@ class BuilderServer(object): @trollius.coroutine def _work_checker(self): while self._current_status == 'running': - logger.debug('Checking for more work') + logger.debug('Checking for more work for %d active workers', self._lifecycle_manager.num_workers()) job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time()) if job_item is None: logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT) diff --git a/conf/init/gunicorn_registry/run b/conf/init/gunicorn_registry/run index a0a09f5a2..3c88fd0e3 100755 --- a/conf/init/gunicorn_registry/run +++ b/conf/init/gunicorn_registry/run @@ -3,6 +3,6 @@ echo 'Starting gunicon' cd / -venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application +nice -n 10 venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/init/gunicorn_verbs/run b/conf/init/gunicorn_verbs/run index 1cf2ee51c..d76a7adcf 100755 --- a/conf/init/gunicorn_verbs/run +++ b/conf/init/gunicorn_verbs/run @@ -3,6 +3,6 @@ echo 'Starting gunicon' cd / -nice -10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application +nice -n 10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/nginx-nossl.conf b/conf/nginx-nossl.conf index fbcce63c0..cc985906a 100644 --- a/conf/nginx-nossl.conf +++ b/conf/nginx-nossl.conf @@ -1,11 +1,5 @@ include root-base.conf; -worker_processes 2; - -user root nogroup; - -daemon off; - http { include http-base.conf; diff --git a/conf/nginx.conf b/conf/nginx.conf index e208d30e0..01d554ae2 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,11 +1,5 @@ include root-base.conf; -worker_processes 2; - -user root nogroup; - -daemon off; - http { include http-base.conf; diff --git a/conf/root-base.conf b/conf/root-base.conf index be8072945..dc8685c34 100644 --- a/conf/root-base.conf +++ b/conf/root-base.conf @@ -1,7 +1,15 @@ pid /tmp/nginx.pid; error_log /var/log/nginx/nginx.error.log; +worker_processes 2; +worker_priority -10; +worker_rlimit_nofile 10240; + +user root nogroup; + +daemon off; + events { - worker_connections 1024; + worker_connections 10240; accept_mutex off; } diff --git a/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py new file mode 100644 index 000000000..a117fd2b1 --- /dev/null +++ b/data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py @@ -0,0 +1,24 @@ +"""Convert slack webhook data + +Revision ID: 5b84373e5db +Revises: 1c5b738283a5 +Create Date: 2014-12-16 12:02:55.167744 + +""" + +# revision identifiers, used by Alembic. +revision = '5b84373e5db' +down_revision = '1c5b738283a5' + +from alembic import op +import sqlalchemy as sa + +from util.migrateslackwebhook import run_slackwebhook_migration + + +def upgrade(tables): + run_slackwebhook_migration() + + +def downgrade(tables): + pass diff --git a/endpoints/notificationmethod.py b/endpoints/notificationmethod.py index d7085f1f0..4d2d685f8 100644 --- a/endpoints/notificationmethod.py +++ b/endpoints/notificationmethod.py @@ -1,14 +1,10 @@ import logging -import io -import os.path -import tarfile -import base64 import json import requests import re from flask.ext.mail import Message -from app import mail, app, get_app_url +from app import mail, app from data import model from workers.worker import JobException @@ -363,11 +359,8 @@ class SlackMethod(NotificationMethod): return 'slack' def validate(self, repository, config_data): - if not config_data.get('token', ''): - raise CannotValidateNotificationMethodException('Missing Slack Token') - - if not config_data.get('subdomain', '').isalnum(): - raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name') + if not config_data.get('url', ''): + raise CannotValidateNotificationMethodException('Missing Slack Callback URL') def format_for_slack(self, message): message = message.replace('\n', '') @@ -378,10 +371,8 @@ class SlackMethod(NotificationMethod): def perform(self, notification, event_handler, notification_data): config_data = json.loads(notification.config_json) - token = config_data.get('token', '') - subdomain = config_data.get('subdomain', '') - - if not token or not subdomain: + url = config_data.get('url', '') + if not url: return owner = model.get_user_or_org(notification.repository.namespace_user.username) @@ -389,8 +380,6 @@ class SlackMethod(NotificationMethod): # Something went wrong. return - url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token) - level = event_handler.get_level(notification_data['event_data'], notification_data) color = { 'info': '#ffffff', @@ -426,5 +415,5 @@ class SlackMethod(NotificationMethod): raise NotificationMethodPerformException(error_message) except requests.exceptions.RequestException as ex: - logger.exception('Slack method was unable to be sent: %s' % ex.message) + logger.exception('Slack method was unable to be sent: %s', ex.message) raise NotificationMethodPerformException(ex.message) diff --git a/endpoints/web.py b/endpoints/web.py index 4717f7d40..913a6905a 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -19,7 +19,7 @@ from util.cache import no_cache from endpoints.common import common_login, render_page_template, route_show_if, param_required from endpoints.csrf import csrf_protect, generate_csrf_token from endpoints.registry import set_cache_headers -from util.names import parse_repository_name +from util.names import parse_repository_name, parse_repository_name_and_tag from util.useremails import send_email_changed from auth import scopes @@ -224,14 +224,14 @@ def robots(): @web.route('/') @no_cache @process_oauth -@parse_repository_name -def redirect_to_repository(namespace, reponame): +@parse_repository_name_and_tag +def redirect_to_repository(namespace, reponame, tag): permission = ReadRepositoryPermission(namespace, reponame) is_public = model.repository_is_public(namespace, reponame) if permission.can() or is_public: repository_name = '/'.join([namespace, reponame]) - return redirect(url_for('web.repository', path=repository_name)) + return redirect(url_for('web.repository', path=repository_name, tag=tag)) abort(404) diff --git a/static/directives/create-external-notification-dialog.html b/static/directives/create-external-notification-dialog.html index 650862690..03479015b 100644 --- a/static/directives/create-external-notification-dialog.html +++ b/static/directives/create-external-notification-dialog.html @@ -73,7 +73,7 @@
- {{ field.title }}: + {{ field.title }}:
@@ -81,6 +81,9 @@ +