Merge remote-tracking branch 'origin/master' into ephemeral

Dockerfile.buildworker

@@ -1,23 +1,30 @@
 # vim:ft=dockerfile
+
+###############################
+# BEGIN COMMON SECION
+###############################
+
 FROM phusion/baseimage:0.9.15
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV HOME /root
 
 # Install the dependencies.
-RUN apt-get update  # 20NOV2014
+RUN apt-get update  # 11DEC2014
 
 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands
-RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev
+RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev
 
 # Build the python dependencies
 ADD requirements.txt requirements.txt
 RUN virtualenv --distribute venv
 RUN venv/bin/pip install -r requirements.txt
 
-RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev
+RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev
 
-### End common section ###
+###############################
+# END COMMON SECION
+###############################
 
 RUN apt-get install -y lxc aufs-tools
 

Dockerfile.web

@@ -1,4 +1,9 @@
 # vim:ft=dockerfile
+
+###############################
+# BEGIN COMMON SECION
+###############################
+
 FROM phusion/baseimage:0.9.15
 
 ENV DEBIAN_FRONTEND noninteractive
@@ -8,16 +13,18 @@ ENV HOME /root
 RUN apt-get update  # 11DEC2014
 
 # New ubuntu packages should be added as their own apt-get install lines below the existing install commands
-RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev
+RUN apt-get install -y git python-virtualenv python-dev libjpeg8 libjpeg62 libjpeg62-dev libevent-2.0.5 libevent-dev gdebi-core g++ libmagic1 phantomjs nodejs npm libldap-2.4-2 libldap2-dev libsasl2-modules libsasl2-dev libpq5 libpq-dev libfreetype6-dev libffi-dev
 
 # Build the python dependencies
 ADD requirements.txt requirements.txt
 RUN virtualenv --distribute venv
 RUN venv/bin/pip install -r requirements.txt
 
-RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev
+RUN apt-get remove -y --auto-remove python-dev g++ libjpeg62-dev libevent-dev libldap2-dev libsasl2-dev libpq-dev libffi-dev
 
-### End common section ###
+###############################
+# END COMMON SECION
+###############################
 
 # Remove SSH.
 RUN rm -rf /etc/service/sshd /etc/my_init.d/00_regen_ssh_host_keys.sh

buildman/builder.py

@@ -49,8 +49,8 @@ def run_build_manager():
   if os.environ.get('SSL_CONFIG'):
     logger.debug('Loading SSL cert and key')
     ssl_context = SSLContext()
-    ssl_context.load_cert_chain(os.environ.get('SSL_CONFIG') + '/ssl.cert',
-                                os.environ.get('SSL_CONFIG') + '/ssl.key')
+    ssl_context.load_cert_chain(os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.cert'),
+                                os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.key'))
 
   server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
                          user_files, manager_klass, build_manager_config[1], public_ip)

buildman/manager/basemanager.py

@@ -59,3 +59,9 @@ class BaseManager(object):
         automatically requeued.
     """
     raise NotImplementedError
+
+  def num_workers(self):
+    """ Returns the number of active build workers currently registered. This includes those
+        that are currently busy and awaiting more work.
+    """
+    raise NotImplementedError

buildman/manager/enterprise.py

@@ -74,3 +74,5 @@ class EnterpriseManager(BaseManager):
     if build_component in self.ready_components:
       self.ready_components.remove(build_component)
 
+  def num_workers(self):
+    return len(self.build_components)

buildman/server.py

@@ -9,7 +9,7 @@ from aiowsgi import create_server as create_wsgi_server
 from flask import Flask
 from threading import Event
 from trollius.coroutines import From
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data.queue import WorkQueue
@@ -140,7 +140,7 @@ class BuilderServer(object):
   @trollius.coroutine
   def _work_checker(self):
     while self._current_status == 'running':
-      logger.debug('Checking for more work')
+      logger.debug('Checking for more work for %d active workers', self._lifecycle_manager.num_workers())
       job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time())
       if job_item is None:
         logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)

conf/init/gunicorn_registry/run

@@ -3,6 +3,6 @@
 echo 'Starting gunicon'
 
 cd /
-venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application
+nice -n 10 venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application
 
 echo 'Gunicorn exited'

conf/init/gunicorn_verbs/run

@@ -3,6 +3,6 @@
 echo 'Starting gunicon'
 
 cd /
-nice -10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application
+nice -n 10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application
 
 echo 'Gunicorn exited'

conf/nginx-nossl.conf

@@ -1,11 +1,5 @@
 include root-base.conf;
 
-worker_processes 2;
-
-user root nogroup;
-
-daemon off;
-
 http {
     include http-base.conf;
 

conf/nginx.conf

@@ -1,11 +1,5 @@
 include root-base.conf;
 
-worker_processes 2;
-
-user root nogroup;
-
-daemon off;
-
 http {
     include http-base.conf;
 

conf/root-base.conf

@@ -1,7 +1,15 @@
 pid /tmp/nginx.pid;
 error_log /var/log/nginx/nginx.error.log;
 
+worker_processes 2;
+worker_priority -10;
+worker_rlimit_nofile 10240;
+
+user root nogroup;
+
+daemon off;
+
 events {
-    worker_connections 1024;
+    worker_connections 10240;
     accept_mutex off;
 }

data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py

@@ -0,0 +1,24 @@
+"""Convert slack webhook data
+
+Revision ID: 5b84373e5db
+Revises: 1c5b738283a5
+Create Date: 2014-12-16 12:02:55.167744
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5b84373e5db'
+down_revision = '1c5b738283a5'
+
+from alembic import op
+import sqlalchemy as sa
+
+from util.migrateslackwebhook import run_slackwebhook_migration
+
+
+def upgrade(tables):
+    run_slackwebhook_migration()
+
+
+def downgrade(tables):
+    pass

endpoints/notificationmethod.py

@@ -1,14 +1,10 @@
 import logging
-import io
-import os.path
-import tarfile
-import base64
 import json
 import requests
 import re
 
 from flask.ext.mail import Message
-from app import mail, app, get_app_url
+from app import mail, app
 from data import model
 from workers.worker import JobException
 
@@ -363,11 +359,8 @@ class SlackMethod(NotificationMethod):
     return 'slack'
 
   def validate(self, repository, config_data):
-    if not config_data.get('token', ''):
-      raise CannotValidateNotificationMethodException('Missing Slack Token')
-
-    if not config_data.get('subdomain', '').isalnum():
-      raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name')
+    if not config_data.get('url', ''):
+      raise CannotValidateNotificationMethodException('Missing Slack Callback URL')
 
   def format_for_slack(self, message):
     message = message.replace('\n', '')
@@ -378,10 +371,8 @@ class SlackMethod(NotificationMethod):
   def perform(self, notification, event_handler, notification_data):
     config_data = json.loads(notification.config_json)
 
-    token = config_data.get('token', '')
-    subdomain = config_data.get('subdomain', '')
-
-    if not token or not subdomain:
+    url = config_data.get('url', '')
+    if not url:
       return
 
     owner = model.get_user_or_org(notification.repository.namespace_user.username)
@@ -389,8 +380,6 @@ class SlackMethod(NotificationMethod):
       # Something went wrong.
       return
 
-    url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token)
-
     level = event_handler.get_level(notification_data['event_data'], notification_data)
     color = {
       'info': '#ffffff',
@@ -426,5 +415,5 @@ class SlackMethod(NotificationMethod):
         raise NotificationMethodPerformException(error_message)
 
     except requests.exceptions.RequestException as ex:
-      logger.exception('Slack method was unable to be sent: %s' % ex.message)
+      logger.exception('Slack method was unable to be sent: %s', ex.message)
       raise NotificationMethodPerformException(ex.message)

endpoints/web.py

@@ -19,7 +19,7 @@ from util.cache import no_cache
 from endpoints.common import common_login, render_page_template, route_show_if, param_required
 from endpoints.csrf import csrf_protect, generate_csrf_token
 from endpoints.registry import set_cache_headers
-from util.names import parse_repository_name
+from util.names import parse_repository_name, parse_repository_name_and_tag
 from util.useremails import send_email_changed
 from auth import scopes
 
@@ -224,14 +224,14 @@ def robots():
 @web.route('/<path:repository>')
 @no_cache
 @process_oauth
-@parse_repository_name
-def redirect_to_repository(namespace, reponame):
+@parse_repository_name_and_tag
+def redirect_to_repository(namespace, reponame, tag):
   permission = ReadRepositoryPermission(namespace, reponame)
   is_public = model.repository_is_public(namespace, reponame)
 
   if permission.can() or is_public:
     repository_name = '/'.join([namespace, reponame])
-    return redirect(url_for('web.repository', path=repository_name))
+    return redirect(url_for('web.repository', path=repository_name, tag=tag))
 
   abort(404)
 

static/directives/create-external-notification-dialog.html

@@ -73,7 +73,7 @@
             <tr ng-if="currentMethod.fields.length"><td colspan="2"><hr></td></tr>
 
             <tr ng-repeat="field in currentMethod.fields">
-              <td valign="top">{{ field.title }}:</td>
+              <td valign="top" style="padding-top: 10px">{{ field.title }}:</td>
               <td>
                 <div ng-switch on="field.type">
                   <span ng-switch-when="email">
@@ -81,6 +81,9 @@
                   </span>
                   <input type="url" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="url" required>
                   <input type="text" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="string" required>
+                  <input type="text" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="regex" required
+                         ng-pattern="getPattern(field)"
+                         placeholder="{{ field.placeholder }}">
                   <div class="entity-search" namespace="repository.namespace"
                        placeholder="''"
                        current-entity="currentConfig[field.name]"

static/js/app.js

@@ -1484,15 +1484,12 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         'icon': 'slack-icon',
         'fields': [
           {
-            'name': 'subdomain',
-            'type': 'string',
-            'title': 'Slack Subdomain'
-          },
-          {
-            'name': 'token',
-            'type': 'string',
-            'title': 'Token',
-            'help_url': 'https://{subdomain}.slack.com/services/new/incoming-webhook'
+            'name': 'url',
+            'type': 'regex',
+            'title': 'Webhook URL',
+            'regex': '^https://hooks\\.slack\\.com/services/[A-Z0-9]+/[A-Z0-9]+/[a-zA-Z0-9]+$',
+            'help_url': 'https://slack.com/services/new/incoming-webhook',
+            'placeholder': 'https://hooks.slack.com/service/{some}/{token}/{here}'
           }
         ]
       }
@@ -5905,6 +5902,10 @@ quayApp.directive('createExternalNotificationDialog', function () {
       $scope.events = ExternalNotificationData.getSupportedEvents();
       $scope.methods = ExternalNotificationData.getSupportedMethods();
 
+      $scope.getPattern = function(field) {
+        return new RegExp(field.regex);
+      };
+
       $scope.setEvent = function(event) {
         $scope.currentEvent = event;
       };

util/migrateslackwebhook.py

@@ -0,0 +1,52 @@
+import logging
+import json
+
+from app import app
+from data.database import configure, RepositoryNotification, ExternalNotificationMethod
+
+configure(app.config)
+
+logger = logging.getLogger(__name__)
+
+def run_slackwebhook_migration():
+  slack_method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == "slack")
+
+  encountered = set()
+  while True:
+    found = list(RepositoryNotification.select().where(
+                      RepositoryNotification.method == slack_method,
+                      RepositoryNotification.config_json ** "%subdomain%",
+                      ~(RepositoryNotification.config_json ** "%url%")))
+
+    found = [f for f in found if not f.uuid in encountered]
+
+    if not found:
+      logger.debug('No additional records found')
+      return
+
+    logger.debug('Found %s records to be changed', len(found))
+    for notification in found:
+      encountered.add(notification.uuid)
+
+      try:
+        config = json.loads(notification.config_json)
+      except:
+        logging.error("Cannot parse config for noticification %s", notification.uuid)
+        continue
+
+      logger.debug("Checking notification %s", notification.uuid)
+      if 'subdomain' in config and 'token' in config:
+        subdomain = config['subdomain']
+        token = config['token']
+        new_url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token)
+        config['url'] = new_url
+
+        logger.debug("Updating notification %s to URL: %s", notification.uuid, new_url)
+        notification.config_json = json.dumps(config)
+        notification.save()
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.DEBUG)
+  logging.getLogger('boto').setLevel(logging.CRITICAL)
+
+  run_slackwebhook_migration()

util/names.py

@@ -4,16 +4,25 @@ from functools import wraps
 from uuid import uuid4
 
 
-def parse_namespace_repository(repository):
+def parse_namespace_repository(repository, tag=False):
   parts = repository.rstrip('/').split('/', 1)
   if len(parts) < 2:
     namespace = 'library'
     repository = parts[0]
   else:
     (namespace, repository) = parts
-  repository = urllib.quote_plus(repository)
-  return (namespace, repository)
 
+  if tag:
+    parts = repository.split(':', 1)
+    if len(parts) < 2:
+      tag = None
+    else:
+      (repository, tag) = parts
+
+  repository = urllib.quote_plus(repository)
+  if tag:
+    return (namespace, repository, tag)
+  return (namespace, repository)
 
 def parse_repository_name(f):
   @wraps(f)
@@ -22,6 +31,13 @@ def parse_repository_name(f):
     return f(namespace, repository, *args, **kwargs)
   return wrapper
 
+def parse_repository_name_and_tag(f):
+  @wraps(f)
+  def wrapper(repository, *args, **kwargs):
+    (namespace, repository, tag) = parse_namespace_repository(repository, tag=True)
+    return f(namespace, repository, tag, *args, **kwargs)
+  return wrapper
+
 
 def format_robot_username(parent_username, robot_shortname):
   return '%s+%s' % (parent_username, robot_shortname)