Merge branch 'master' of https://github.com/coreos-inc/quay

buildman/builder.py

@@ -44,8 +44,8 @@ def run_build_manager():
   if os.environ.get('SSL_CONFIG'):
     logger.debug('Loading SSL cert and key')
     ssl_context = SSLContext()
-    ssl_context.load_cert_chain(os.environ.get('SSL_CONFIG') + '/ssl.cert',
-                                os.environ.get('SSL_CONFIG') + '/ssl.key')
+    ssl_context.load_cert_chain(os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.cert'),
+                                os.path.join(os.environ.get('SSL_CONFIG'), 'ssl.key'))
 
   server = BuilderServer(app.config['SERVER_HOSTNAME'], dockerfile_build_queue, build_logs,
                          user_files, manager_klass)

buildman/manager/basemanager.py

@@ -47,3 +47,9 @@ class BaseManager(object):
         one of: incomplete, error, complete. If incomplete, the job should be requeued.
     """
     raise NotImplementedError
+
+  def num_workers(self):
+    """ Returns the number of active build workers currently registered. This includes those
+        that are currently busy and awaiting more work.
+    """
+    raise NotImplementedError

buildman/manager/enterprise.py

@@ -70,3 +70,5 @@ class EnterpriseManager(BaseManager):
   def build_component_disposed(self, build_component, timed_out):
     self.build_components.remove(build_component)
 
+  def num_workers(self):
+    return len(self.build_components)

buildman/server.py

@@ -9,7 +9,7 @@ from aiowsgi import create_server as create_wsgi_server
 from flask import Flask
 from threading import Event
 from trollius.coroutines import From
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from buildman.jobutil.buildjob import BuildJob, BuildJobLoadException
 from data.queue import WorkQueue
@@ -137,7 +137,7 @@ class BuilderServer(object):
   @trollius.coroutine
   def _work_checker(self):
     while self._current_status == 'running':
-      logger.debug('Checking for more work')
+      logger.debug('Checking for more work for %d active workers', self._lifecycle_manager.num_workers())
       job_item = self._queue.get(processing_time=self._lifecycle_manager.setup_time())
       if job_item is None:
         logger.debug('No additional work found. Going to sleep for %s seconds', WORK_CHECK_TIMEOUT)

data/migrations/versions/5b84373e5db_convert_slack_webhook_data.py

@@ -0,0 +1,24 @@
+"""Convert slack webhook data
+
+Revision ID: 5b84373e5db
+Revises: 1c5b738283a5
+Create Date: 2014-12-16 12:02:55.167744
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5b84373e5db'
+down_revision = '1c5b738283a5'
+
+from alembic import op
+import sqlalchemy as sa
+
+from tools.migrateslackwebhook import run_slackwebhook_migration
+
+
+def upgrade(tables):
+    run_slackwebhook_migration()
+
+
+def downgrade(tables):
+    pass

endpoints/notificationmethod.py

@@ -1,14 +1,10 @@
 import logging
-import io
-import os.path
-import tarfile
-import base64
 import json
 import requests
 import re
 
 from flask.ext.mail import Message
-from app import mail, app, get_app_url
+from app import mail, app
 from data import model
 from workers.worker import JobException
 
@@ -363,11 +359,8 @@ class SlackMethod(NotificationMethod):
     return 'slack'
 
   def validate(self, repository, config_data):
-    if not config_data.get('token', ''):
-      raise CannotValidateNotificationMethodException('Missing Slack Token')
-
-    if not config_data.get('subdomain', '').isalnum():
-      raise CannotValidateNotificationMethodException('Missing Slack Subdomain Name')
+    if not config_data.get('url', ''):
+      raise CannotValidateNotificationMethodException('Missing Slack Callback URL')
 
   def format_for_slack(self, message):
     message = message.replace('\n', '')
@@ -378,10 +371,8 @@ class SlackMethod(NotificationMethod):
   def perform(self, notification, event_handler, notification_data):
     config_data = json.loads(notification.config_json)
 
-    token = config_data.get('token', '')
-    subdomain = config_data.get('subdomain', '')
-
-    if not token or not subdomain:
+    url = config_data.get('url', '')
+    if not url:
       return
 
     owner = model.get_user_or_org(notification.repository.namespace_user.username)
@@ -389,8 +380,6 @@ class SlackMethod(NotificationMethod):
       # Something went wrong.
       return
 
-    url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token)
-
     level = event_handler.get_level(notification_data['event_data'], notification_data)
     color = {
       'info': '#ffffff',
@@ -426,5 +415,5 @@ class SlackMethod(NotificationMethod):
         raise NotificationMethodPerformException(error_message)
 
     except requests.exceptions.RequestException as ex:
-      logger.exception('Slack method was unable to be sent: %s' % ex.message)
+      logger.exception('Slack method was unable to be sent: %s', ex.message)
       raise NotificationMethodPerformException(ex.message)

static/directives/create-external-notification-dialog.html

@@ -73,7 +73,7 @@
             <tr ng-if="currentMethod.fields.length"><td colspan="2"><hr></td></tr>
 
             <tr ng-repeat="field in currentMethod.fields">
-              <td valign="top">{{ field.title }}:</td>
+              <td valign="top" style="padding-top: 10px">{{ field.title }}:</td>
               <td>
                 <div ng-switch on="field.type">
                   <span ng-switch-when="email">
@@ -81,6 +81,9 @@
                   </span>
                   <input type="url" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="url" required>
                   <input type="text" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="string" required>
+                  <input type="text" class="form-control" ng-model="currentConfig[field.name]"  ng-switch-when="regex" required
+                         ng-pattern="getPattern(field)"
+                         placeholder="{{ field.placeholder }}">
                   <div class="entity-search" namespace="repository.namespace"
                        placeholder="''"
                        current-entity="currentConfig[field.name]"

static/js/app.js

@@ -1484,15 +1484,12 @@ quayApp = angular.module('quay', quayDependencies, function($provide, cfpLoading
         'icon': 'slack-icon',
         'fields': [
           {
-            'name': 'subdomain',
-            'type': 'string',
-            'title': 'Slack Subdomain'
-          },
-          {
-            'name': 'token',
-            'type': 'string',
-            'title': 'Token',
-            'help_url': 'https://{subdomain}.slack.com/services/new/incoming-webhook'
+            'name': 'url',
+            'type': 'regex',
+            'title': 'Webhook URL',
+            'regex': '^https://hooks\\.slack\\.com/services/[A-Z0-9]+/[A-Z0-9]+/[a-zA-Z0-9]+$',
+            'help_url': 'https://slack.com/services/new/incoming-webhook',
+            'placeholder': 'https://hooks.slack.com/service/{some}/{token}/{here}'
           }
         ]
       }
@@ -5905,6 +5902,10 @@ quayApp.directive('createExternalNotificationDialog', function () {
       $scope.events = ExternalNotificationData.getSupportedEvents();
       $scope.methods = ExternalNotificationData.getSupportedMethods();
 
+      $scope.getPattern = function(field) {
+        return new RegExp(field.regex);
+      };
+
       $scope.setEvent = function(event) {
         $scope.currentEvent = event;
       };

tools/migrateslackwebhook.py

@@ -0,0 +1,52 @@
+import logging
+import json
+
+from app import app
+from data.database import configure, RepositoryNotification, ExternalNotificationMethod
+
+configure(app.config)
+
+logger = logging.getLogger(__name__)
+
+def run_slackwebhook_migration():
+  slack_method = ExternalNotificationMethod.get(ExternalNotificationMethod.name == "slack")
+
+  encountered = set()
+  while True:
+    found = list(RepositoryNotification.select().where(
+                      RepositoryNotification.method == slack_method,
+                      RepositoryNotification.config_json ** "%subdomain%",
+                      ~(RepositoryNotification.config_json ** "%url%")))
+
+    found = [f for f in found if not f.uuid in encountered]
+
+    if not found:
+      logger.debug('No additional records found')
+      return
+
+    logger.debug('Found %s records to be changed', len(found))
+    for notification in found:
+      encountered.add(notification.uuid)
+
+      try:
+        config = json.loads(notification.config_json)
+      except:
+        logging.error("Cannot parse config for noticification %s", notification.uuid)
+        continue
+
+      logger.debug("Checking notification %s", notification.uuid)
+      if 'subdomain' in config and 'token' in config:
+        subdomain = config['subdomain']
+        token = config['token']
+        new_url = 'https://%s.slack.com/services/hooks/incoming-webhook?token=%s' % (subdomain, token)
+        config['url'] = new_url
+
+        logger.debug("Updating notification %s to URL: %s", notification.uuid, new_url)
+        notification.config_json = json.dumps(config)
+        notification.save()
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.DEBUG)
+  logging.getLogger('boto').setLevel(logging.CRITICAL)
+
+  run_slackwebhook_migration()