Work In Progress!

Get the full activation and deactivation cycle working for bitbucket.

endpoints/trigger.py

@@ -7,7 +7,7 @@ import re
 import json
 
 from github import Github, UnknownObjectException, GithubException
-from bitbucket.bitbucket import Bitbucket
+from bitbucket import BitBucket
 from tempfile import SpooledTemporaryFile
 from jsonschema import validate
 from data import model
@@ -186,75 +186,123 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
   def service_name(cls):
     return 'bitbucket'
 
-  def _get_authorized_client(self, namespace=None):
+  def _get_client(self):
     key = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_KEY', '')
     secret = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_SECRET', '')
 
     trigger_uuid = self.trigger.uuid
     callback_url = '%s/oauth1/bitbucket/callback/trigger/%s' % (get_app_url(), trigger_uuid)
 
-    bitbucket_client = Bitbucket(username=namespace or self.config.get('username', ''))
+    return BitBucket(key, secret, callback_url)
 
-    (result, err_message) = bitbucket_client.authorize(key, secret, callback_url,
-                                 access_token=self.config.get('access_token'),
-                                 access_token_secret=self.auth_token)
-    if not result:
-      raise TriggerProviderException(err_message)
+  def _get_authorized_client(self):
+    base_client = self._get_client()
+    auth_token = self.auth_token or 'invalid:invalid'
+    (access_token, access_token_secret) = auth_token.split(':')
+    return base_client.get_authorized_client(access_token, access_token_secret)
 
-    return bitbucket_client
+  def _get_repository_client(self):
+    source = self.config['build_source']
+    (namespace, name) = source.split('/')
+    bitbucket_client = self._get_authorized_client()
+    return bitbucket_client.for_namespace(namespace).repositories().get(name)
 
   def get_oauth_url(self):
-    bitbucket_client = self._get_authorized_client()
-    url = bitbucket_client.url('AUTHENTICATE', token=bitbucket_client.access_token)
-    return {
-      'access_token': bitbucket_client.access_token,
-      'access_token_secret': bitbucket_client.access_token_secret,
-      'url': url
-    }
+    bitbucket_client = self._get_client()
+    (result, data, err_msg) = bitbucket_client.get_authorization_url()
+    if not result:
+      raise RepositoryReadException(err_msg)
+
+    return data
 
   def exchange_verifier(self, verifier):
-    bitbucket_client = self._get_authorized_client()
-    (result, data) = bitbucket_client.verify(verifier,
-                                          access_token=self.config.get('access_token', ''),
-                                          access_token_secret=self.auth_token)
+    bitbucket_client = self._get_client()
+    access_token = self.config.get('access_token', '')
+    access_token_secret = self.auth_token
 
+    # Exchange the verifier for a new access token.
+    (result, data, _) = bitbucket_client.verify_token(access_token, access_token_secret, verifier)
     if not result:
       return False
 
-    # Request the user's information and save it and the access token to the config.
-    user_url = bitbucket_client.URLS['BASE'] % 'user'
-    (result, data) = bitbucket_client.dispatch('GET', user_url, auth=bitbucket_client.auth)
+    # Save the updated access token and secret.
+    self.set_auth_token(data[0] + ':' + data[1])
+
+    # Retrieve the current authorized user's information and store the username in the config.
+    authorized_client = self._get_authorized_client()
+    (result, data, _) = authorized_client.get_current_user()
     if not result:
       return False
 
     username = data['user']['username']
-    new_access_token = bitbucket_client.access_token
-
     self.put_config_key('username', username)
-    self.put_config_key('access_token', new_access_token)
-    self.set_auth_token(bitbucket_client.access_token_secret)
-
     return True
 
-
   def is_active(self):
-    return False
+    return 'hook_id' in self.config
 
   def activate(self, standard_webhook_url):
-    return {}
+    config = self.config
+
+    # Add a deploy key to the repository.
+    public_key, private_key = generate_ssh_keypair()
+    config['credentials'] = [
+      {
+        'name': 'SSH Public Key',
+        'value': public_key,
+      },
+    ]
+
+    repository = self._get_repository_client()
+    (result, data, err_msg) = repository.deploykeys().create(
+      app.config['REGISTRY_TITLE'] + ' webhook key', public_key)
+
+    if not result:
+      msg = 'Unable to add deploy key to repository: %s' % err_msg
+      raise TriggerActivationException(msg)
+
+    config['deploy_key_id'] = data['pk']
+
+    # Add a webhook callback.
+    (result, data, err_msg) = repository.services().create('POST', URL=standard_webhook_url)
+    if not result:
+      msg = 'Unable to add webhook to repository: %s' % err_msg
+      raise TriggerActivationException(msg)
+
+    config['hook_id'] = data['id']
+    return config, {'private_key': private_key}
+
 
   def deactivate(self):
-    return self.config
+    config = self.config
+    repository = self._get_repository_client()
+
+    # Remove the webhook link.
+    (result, _, err_msg) = repository.services().delete(config['hook_id'])
+    if not result:
+      msg = 'Unable to remove webhook from repository: %s' % err_msg
+      raise TriggerDeactivationException(msg)
+
+    # Remove the public key.
+    (result, _, err_msg) = repository.deploykeys().delete(config['deploy_key_id'])
+    if not result:
+      msg = 'Unable to remove deploy key from repository: %s' % err_msg
+      raise TriggerDeactivationException(msg)
+
+    config.pop('hook_id', None)
+    config.pop('deploy_key_id', None)
+
+    return config
+
 
   def list_build_sources(self):
     bitbucket_client = self._get_authorized_client()
-    success, repositories = bitbucket_client.repository.all()
-    if not success:
-      raise RepositoryReadException('Could not read repository list')
+    (result, data, err_msg) = bitbucket_client.get_visible_repositories()
+    if not result:
+      raise RepositoryReadException('Could not read repository list: ' + err_msg)
 
     namespaces = {}
-
-    for repo in repositories:
+    for repo in data:
       if not repo['scm'] == 'git':
         continue
 
@@ -272,36 +320,101 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
 
     return namespaces.values()
 
-
   def list_build_subdirs(self):
-    source = self.config['build_source']
-    (namespace, name) = source.split('/')
-    (result, data) = self._get_authorized_client(namespace=namespace).repository.get(name)
+    repository = self._get_repository_client()
+    (result, data, err_msg) = repository.get_path_contents('', revision='master')
+    if not result:
+      raise RepositoryReadException(err_msg)
+
+
+    files = set([f['path'] for f in data['files']])
+    if 'Dockerfile' in files:
+      return ['/']
 
-    print result
-    print data
     return []
 
   def dockerfile_url(self):
-    return None
+    repository = self._get_repository_client()
+    subdirectory = self.config.get('subdir', '')
+    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
+
+    master_branch = 'master'
+    (result, data, _) = repository.get_main_branch()
+    if result:
+      master_branch = data['name']
+
+    return 'https://bitbucket.org/%s/%s/src/%s/%s' % (repository.namespace,
+                                                      repository.repository_name,
+                                                      master_branch, path)
 
   def load_dockerfile_contents(self):
-    raise RepositoryReadException('Not supported')
+    repository = self._get_repository_client()
+    subdirectory = self.config.get('subdir', '/')[1:]
+    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
 
-  def handle_trigger_request(self, request):
-    return
+    (result, data, err_msg) = repository.get_raw_path_contents(path, revision='master')
+    if not result:
+      raise RepositoryReadException(err_msg)
 
-  def manual_start(self, run_parameters=None):
-    return None
+    return data
 
   def list_field_values(self, field_name):
     source = self.config['build_source']
     (namespace, name) = source.split('/')
-    (result, data) = self._get_authorized_client(namespace=namespace).repository.get(name)
 
-    print result
-    print data
-    return []
+    bitbucket_client = self._get_authorized_client()
+    repository = bitbucket_client.for_namespace(namespace).repositories().get(name)
+
+    if field_name == 'refs':
+      (result, data, _) = repository.get_branches_and_tags()
+      if not result:
+        return None
+
+      branches = [b['name'] for b in data['branches']]
+      tags = [t['name'] for t in data['tags']]
+
+      return ([{'kind': 'branch', 'name': b} for b in branches] +
+              [{'kind': 'tag', 'name': tag} for tag in tags])
+
+    if field_name == 'tag_name':
+      (result, data, _) = repository.get_tags()
+      if not result:
+        return None
+
+      return data.keys()
+
+    if field_name == 'branch_name':
+      (result, data, _) = repository.get_branches()
+      if not result:
+        return None
+
+      return data.keys()
+
+    return None
+
+
+  def handle_trigger_request(self, request):
+    return
+
+
+  def manual_start(self, run_parameters=None):
+    config = self.config
+    repository = self._get_repository_client()
+
+    source = config['build_source']
+    run_parameters = run_parameters or {}
+
+    # Lookup the branch to build.
+    master_branch = 'master'
+    (result, data, _) = repository.get_main_branch()
+    if result:
+      master_branch = data['name']
+
+    branch_name = run_parameters.get('branch_name') or master_branch
+
+    # Find the SHA for the branch.
+    # TODO
+    return None
 
 
 class GithubBuildTrigger(BuildTriggerHandler):

requirements-nover.txt

@@ -42,6 +42,7 @@ git+https://github.com/DevTable/avatar-generator.git
 git+https://github.com/DevTable/pygithub.git
 git+https://github.com/DevTable/container-cloud-config.git
 git+https://github.com/DevTable/python-etcd.git
+git+https://github.com/coreos/py-bitbucket.git
 gipc
 pyOpenSSL
 pygpgme

requirements.txt

@@ -69,3 +69,4 @@ git+https://github.com/DevTable/pygithub.git
 git+https://github.com/DevTable/container-cloud-config.git
 git+https://github.com/DevTable/python-etcd.git
 git+https://github.com/NateFerrero/oauth2lib.git
+git+https://github.com/coreos/py-bitbucket.git

static/directives/dropdown-select.html

@@ -4,7 +4,7 @@
     <input type="text" class="lookahead-input form-control" placeholder="{{ placeholder }}"
            ng-readonly="!allowCustomInput"></input>
   </div>
-  <div class="dropdown">
+  <div class="dropdown" ng-show="!hideDropdown">
     <button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown">
       <span class="caret"></span>
     </button>

static/directives/trigger-setup-githost.html

@@ -164,7 +164,8 @@
       <div class="dropdown-select" placeholder="'(Repository Root)'" selected-item="state.currentLocation"
            lookahead-items="locations" handle-input="handleLocationInput(input)"
            handle-item-selected="handleLocationSelected(datum)"
-           allow-custom-input="true">
+           allow-custom-input="true"
+           hide-dropdown="!supportsFullListing">
         <!-- Icons -->
         <i class="dropdown-select-icon none-icon fa fa-folder-o fa-lg" ng-show="state.isInvalidLocation"></i>
         <i class="dropdown-select-icon none-icon fa fa-folder fa-lg" style="color: black;" ng-show="!state.isInvalidLocation"></i>
@@ -187,13 +188,13 @@
       </div>
 
       <div class="quay-spinner" ng-show="!locations && !locationError"></div>
-      <div class="alert alert-warning" ng-show="locations && !locations.length">
-        Warning: No Dockerfiles were found in {{ state.currentRepo.repo }}
-      </div>
       <div class="alert alert-warning" ng-show="locationError">
         {{ locationError }}
       </div>
-      <div class="alert alert-info" ng-show="locations.length && state.isInvalidLocation">
+      <div class="alert alert-warning" ng-show="locations && !locations.length && supportsFullListing">
+        Warning: No Dockerfiles were found in {{ state.currentRepo.repo }}
+      </div>
+      <div class="alert alert-info" ng-show="locations.length && state.isInvalidLocation && supportsFullListing">
         Note: The folder does not currently exist or contain a Dockerfile
       </div>
     </div>

static/js/directives/ui/dropdown-select.js

@@ -13,6 +13,7 @@ angular.module('quay').directive('dropdownSelect', function ($compile) {
       'selectedItem': '=selectedItem',
       'placeholder': '=placeholder',
       'lookaheadItems': '=lookaheadItems',
+      'hideDropdown': '=hideDropdown',
 
       'allowCustomInput': '@allowCustomInput',
 

static/js/directives/ui/trigger-setup-githost.js

@@ -18,7 +18,7 @@ angular.module('quay').directive('triggerSetupGithost', function () {
 
       'analyze': '&analyze'
     },
-    controller: function($scope, $element, ApiService) {
+    controller: function($scope, $element, ApiService, TriggerService) {
       $scope.analyzeCounter = 0;
       $scope.setupReady = false;
       $scope.refs = null;
@@ -33,6 +33,12 @@ angular.module('quay').directive('triggerSetupGithost', function () {
         'currentLocation': null
       };
 
+      var checkLocation = function() {
+        var location = $scope.state.currentLocation || '';
+        $scope.state.isInvalidLocation = $scope.supportsFullListing &&
+                                         $scope.locations.indexOf(location) < 0;
+      };
+
       $scope.isMatching = function(kind, name, filter) {
         try {
           var patt = new RegExp(filter);
@@ -122,8 +128,8 @@ angular.module('quay').directive('triggerSetupGithost', function () {
             $scope.setLocation($scope.locations[0]);
           } else {
             $scope.state.currentLocation = null;
-            $scope.state.isInvalidLocation = resp['subdir'].indexOf('') < 0;
             $scope.trigger.$ready = true;
+            checkLocation();
           }
 
           callback();
@@ -131,9 +137,9 @@ angular.module('quay').directive('triggerSetupGithost', function () {
       }
 
       $scope.handleLocationInput = function(location) {
-        $scope.state.isInvalidLocation = $scope.locations.indexOf(location) < 0;
         $scope.trigger['config']['subdir'] = location || '';
         $scope.trigger.$ready = true;
+        checkLocation();
       };
 
       $scope.handleLocationSelected = function(datum) {
@@ -142,9 +148,9 @@ angular.module('quay').directive('triggerSetupGithost', function () {
 
       $scope.setLocation = function(location) {
         $scope.state.currentLocation = location;
-        $scope.state.isInvalidLocation = false;
         $scope.trigger['config']['subdir'] = location || '';
         $scope.trigger.$ready = true;
+        checkLocation();
       };
 
       $scope.selectRepo = function(repo, org) {
@@ -199,6 +205,11 @@ angular.module('quay').directive('triggerSetupGithost', function () {
         $scope.repoLookahead = repos;
       };
 
+      $scope.$watch('trigger', function(trigger) {
+        if (!trigger) { return; }
+        $scope.supportsFullListing = TriggerService.supportsFullListing(trigger.service)
+      });
+
       $scope.$watch('state.currentRepo', function(repo) {
         if (repo) {
           $scope.selectRepoInternal(repo);

static/js/services/trigger-service.js

@@ -49,7 +49,8 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
         }
 
         return 'GitHub Repository Push';
-      }
+      },
+      'supports_full_directory_listing': true
     },
 
     'bitbucket': {
@@ -75,7 +76,8 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
         return Features.BITBUCKET_BUILD;
       },
       'icon': 'fa-bitbucket',
-      'title': function() { return 'Bitbucket Repository Push'; }
+      'title': function() { return 'Bitbucket Repository Push'; },
+      'supports_full_directory_listing': false
     },
 
     'custom-git': {
@@ -104,6 +106,15 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
     }
   }
 
+  triggerService.supportsFullListing = function(name) {
+    var type = triggerTypes[name];
+    if (!type) {
+      return false;
+    }
+
+    return !!type['supports_full_directory_listing'];
+  };
+
   triggerService.getTypes = function() {
     var types = [];
     for (var key in triggerTypes) {