From 6479f8ddc9c03d5ae1e4ed1e61719c3388034f71 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 28 Apr 2015 18:15:12 -0400 Subject: [PATCH] Work In Progress! Get the full activation and deactivation cycle working for bitbucket. --- endpoints/trigger.py | 219 +++++++++++++----- requirements-nover.txt | 1 + requirements.txt | 1 + static/directives/dropdown-select.html | 2 +- static/directives/trigger-setup-githost.html | 11 +- static/js/directives/ui/dropdown-select.js | 1 + .../js/directives/ui/trigger-setup-githost.js | 19 +- static/js/services/trigger-service.js | 15 +- 8 files changed, 204 insertions(+), 65 deletions(-) diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 8f00d4674..2882c494f 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -7,7 +7,7 @@ import re import json from github import Github, UnknownObjectException, GithubException -from bitbucket.bitbucket import Bitbucket +from bitbucket import BitBucket from tempfile import SpooledTemporaryFile from jsonschema import validate from data import model @@ -186,75 +186,123 @@ class BitbucketBuildTrigger(BuildTriggerHandler): def service_name(cls): return 'bitbucket' - def _get_authorized_client(self, namespace=None): + def _get_client(self): key = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_KEY', '') secret = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_SECRET', '') trigger_uuid = self.trigger.uuid callback_url = '%s/oauth1/bitbucket/callback/trigger/%s' % (get_app_url(), trigger_uuid) - bitbucket_client = Bitbucket(username=namespace or self.config.get('username', '')) + return BitBucket(key, secret, callback_url) - (result, err_message) = bitbucket_client.authorize(key, secret, callback_url, - access_token=self.config.get('access_token'), - access_token_secret=self.auth_token) - if not result: - raise TriggerProviderException(err_message) + def _get_authorized_client(self): + base_client = self._get_client() + auth_token = self.auth_token or 'invalid:invalid' + (access_token, access_token_secret) = auth_token.split(':') + return base_client.get_authorized_client(access_token, access_token_secret) - return bitbucket_client + def _get_repository_client(self): + source = self.config['build_source'] + (namespace, name) = source.split('/') + bitbucket_client = self._get_authorized_client() + return bitbucket_client.for_namespace(namespace).repositories().get(name) def get_oauth_url(self): - bitbucket_client = self._get_authorized_client() - url = bitbucket_client.url('AUTHENTICATE', token=bitbucket_client.access_token) - return { - 'access_token': bitbucket_client.access_token, - 'access_token_secret': bitbucket_client.access_token_secret, - 'url': url - } + bitbucket_client = self._get_client() + (result, data, err_msg) = bitbucket_client.get_authorization_url() + if not result: + raise RepositoryReadException(err_msg) + + return data def exchange_verifier(self, verifier): - bitbucket_client = self._get_authorized_client() - (result, data) = bitbucket_client.verify(verifier, - access_token=self.config.get('access_token', ''), - access_token_secret=self.auth_token) + bitbucket_client = self._get_client() + access_token = self.config.get('access_token', '') + access_token_secret = self.auth_token + # Exchange the verifier for a new access token. + (result, data, _) = bitbucket_client.verify_token(access_token, access_token_secret, verifier) if not result: return False - # Request the user's information and save it and the access token to the config. - user_url = bitbucket_client.URLS['BASE'] % 'user' - (result, data) = bitbucket_client.dispatch('GET', user_url, auth=bitbucket_client.auth) + # Save the updated access token and secret. + self.set_auth_token(data[0] + ':' + data[1]) + + # Retrieve the current authorized user's information and store the username in the config. + authorized_client = self._get_authorized_client() + (result, data, _) = authorized_client.get_current_user() if not result: return False username = data['user']['username'] - new_access_token = bitbucket_client.access_token - self.put_config_key('username', username) - self.put_config_key('access_token', new_access_token) - self.set_auth_token(bitbucket_client.access_token_secret) - return True - def is_active(self): - return False + return 'hook_id' in self.config def activate(self, standard_webhook_url): - return {} + config = self.config + + # Add a deploy key to the repository. + public_key, private_key = generate_ssh_keypair() + config['credentials'] = [ + { + 'name': 'SSH Public Key', + 'value': public_key, + }, + ] + + repository = self._get_repository_client() + (result, data, err_msg) = repository.deploykeys().create( + app.config['REGISTRY_TITLE'] + ' webhook key', public_key) + + if not result: + msg = 'Unable to add deploy key to repository: %s' % err_msg + raise TriggerActivationException(msg) + + config['deploy_key_id'] = data['pk'] + + # Add a webhook callback. + (result, data, err_msg) = repository.services().create('POST', URL=standard_webhook_url) + if not result: + msg = 'Unable to add webhook to repository: %s' % err_msg + raise TriggerActivationException(msg) + + config['hook_id'] = data['id'] + return config, {'private_key': private_key} + def deactivate(self): - return self.config + config = self.config + repository = self._get_repository_client() + + # Remove the webhook link. + (result, _, err_msg) = repository.services().delete(config['hook_id']) + if not result: + msg = 'Unable to remove webhook from repository: %s' % err_msg + raise TriggerDeactivationException(msg) + + # Remove the public key. + (result, _, err_msg) = repository.deploykeys().delete(config['deploy_key_id']) + if not result: + msg = 'Unable to remove deploy key from repository: %s' % err_msg + raise TriggerDeactivationException(msg) + + config.pop('hook_id', None) + config.pop('deploy_key_id', None) + + return config + def list_build_sources(self): bitbucket_client = self._get_authorized_client() - success, repositories = bitbucket_client.repository.all() - if not success: - raise RepositoryReadException('Could not read repository list') + (result, data, err_msg) = bitbucket_client.get_visible_repositories() + if not result: + raise RepositoryReadException('Could not read repository list: ' + err_msg) namespaces = {} - - for repo in repositories: + for repo in data: if not repo['scm'] == 'git': continue @@ -272,36 +320,101 @@ class BitbucketBuildTrigger(BuildTriggerHandler): return namespaces.values() - def list_build_subdirs(self): - source = self.config['build_source'] - (namespace, name) = source.split('/') - (result, data) = self._get_authorized_client(namespace=namespace).repository.get(name) + repository = self._get_repository_client() + (result, data, err_msg) = repository.get_path_contents('', revision='master') + if not result: + raise RepositoryReadException(err_msg) + + + files = set([f['path'] for f in data['files']]) + if 'Dockerfile' in files: + return ['/'] - print result - print data return [] def dockerfile_url(self): - return None + repository = self._get_repository_client() + subdirectory = self.config.get('subdir', '') + path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile' + + master_branch = 'master' + (result, data, _) = repository.get_main_branch() + if result: + master_branch = data['name'] + + return 'https://bitbucket.org/%s/%s/src/%s/%s' % (repository.namespace, + repository.repository_name, + master_branch, path) def load_dockerfile_contents(self): - raise RepositoryReadException('Not supported') + repository = self._get_repository_client() + subdirectory = self.config.get('subdir', '/')[1:] + path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile' - def handle_trigger_request(self, request): - return + (result, data, err_msg) = repository.get_raw_path_contents(path, revision='master') + if not result: + raise RepositoryReadException(err_msg) - def manual_start(self, run_parameters=None): - return None + return data def list_field_values(self, field_name): source = self.config['build_source'] (namespace, name) = source.split('/') - (result, data) = self._get_authorized_client(namespace=namespace).repository.get(name) - print result - print data - return [] + bitbucket_client = self._get_authorized_client() + repository = bitbucket_client.for_namespace(namespace).repositories().get(name) + + if field_name == 'refs': + (result, data, _) = repository.get_branches_and_tags() + if not result: + return None + + branches = [b['name'] for b in data['branches']] + tags = [t['name'] for t in data['tags']] + + return ([{'kind': 'branch', 'name': b} for b in branches] + + [{'kind': 'tag', 'name': tag} for tag in tags]) + + if field_name == 'tag_name': + (result, data, _) = repository.get_tags() + if not result: + return None + + return data.keys() + + if field_name == 'branch_name': + (result, data, _) = repository.get_branches() + if not result: + return None + + return data.keys() + + return None + + + def handle_trigger_request(self, request): + return + + + def manual_start(self, run_parameters=None): + config = self.config + repository = self._get_repository_client() + + source = config['build_source'] + run_parameters = run_parameters or {} + + # Lookup the branch to build. + master_branch = 'master' + (result, data, _) = repository.get_main_branch() + if result: + master_branch = data['name'] + + branch_name = run_parameters.get('branch_name') or master_branch + + # Find the SHA for the branch. + # TODO + return None class GithubBuildTrigger(BuildTriggerHandler): diff --git a/requirements-nover.txt b/requirements-nover.txt index 055d38875..f6321c322 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -42,6 +42,7 @@ git+https://github.com/DevTable/avatar-generator.git git+https://github.com/DevTable/pygithub.git git+https://github.com/DevTable/container-cloud-config.git git+https://github.com/DevTable/python-etcd.git +git+https://github.com/coreos/py-bitbucket.git gipc pyOpenSSL pygpgme diff --git a/requirements.txt b/requirements.txt index 0767cc1af..8c3ca0f2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,3 +69,4 @@ git+https://github.com/DevTable/pygithub.git git+https://github.com/DevTable/container-cloud-config.git git+https://github.com/DevTable/python-etcd.git git+https://github.com/NateFerrero/oauth2lib.git +git+https://github.com/coreos/py-bitbucket.git diff --git a/static/directives/dropdown-select.html b/static/directives/dropdown-select.html index 49047564d..333d27f5c 100644 --- a/static/directives/dropdown-select.html +++ b/static/directives/dropdown-select.html @@ -4,7 +4,7 @@ -
+
diff --git a/static/directives/trigger-setup-githost.html b/static/directives/trigger-setup-githost.html index 67e44a5be..d163a04d6 100644 --- a/static/directives/trigger-setup-githost.html +++ b/static/directives/trigger-setup-githost.html @@ -164,7 +164,8 @@
+ allow-custom-input="true" + hide-dropdown="!supportsFullListing"> @@ -187,13 +188,13 @@
-
- Warning: No Dockerfiles were found in {{ state.currentRepo.repo }} -
{{ locationError }}
-
+
+ Warning: No Dockerfiles were found in {{ state.currentRepo.repo }} +
+
Note: The folder does not currently exist or contain a Dockerfile
diff --git a/static/js/directives/ui/dropdown-select.js b/static/js/directives/ui/dropdown-select.js index 2ca86478e..ff84b162b 100644 --- a/static/js/directives/ui/dropdown-select.js +++ b/static/js/directives/ui/dropdown-select.js @@ -13,6 +13,7 @@ angular.module('quay').directive('dropdownSelect', function ($compile) { 'selectedItem': '=selectedItem', 'placeholder': '=placeholder', 'lookaheadItems': '=lookaheadItems', + 'hideDropdown': '=hideDropdown', 'allowCustomInput': '@allowCustomInput', diff --git a/static/js/directives/ui/trigger-setup-githost.js b/static/js/directives/ui/trigger-setup-githost.js index 7633b2811..d3031bc62 100644 --- a/static/js/directives/ui/trigger-setup-githost.js +++ b/static/js/directives/ui/trigger-setup-githost.js @@ -18,7 +18,7 @@ angular.module('quay').directive('triggerSetupGithost', function () { 'analyze': '&analyze' }, - controller: function($scope, $element, ApiService) { + controller: function($scope, $element, ApiService, TriggerService) { $scope.analyzeCounter = 0; $scope.setupReady = false; $scope.refs = null; @@ -33,6 +33,12 @@ angular.module('quay').directive('triggerSetupGithost', function () { 'currentLocation': null }; + var checkLocation = function() { + var location = $scope.state.currentLocation || ''; + $scope.state.isInvalidLocation = $scope.supportsFullListing && + $scope.locations.indexOf(location) < 0; + }; + $scope.isMatching = function(kind, name, filter) { try { var patt = new RegExp(filter); @@ -122,8 +128,8 @@ angular.module('quay').directive('triggerSetupGithost', function () { $scope.setLocation($scope.locations[0]); } else { $scope.state.currentLocation = null; - $scope.state.isInvalidLocation = resp['subdir'].indexOf('') < 0; $scope.trigger.$ready = true; + checkLocation(); } callback(); @@ -131,9 +137,9 @@ angular.module('quay').directive('triggerSetupGithost', function () { } $scope.handleLocationInput = function(location) { - $scope.state.isInvalidLocation = $scope.locations.indexOf(location) < 0; $scope.trigger['config']['subdir'] = location || ''; $scope.trigger.$ready = true; + checkLocation(); }; $scope.handleLocationSelected = function(datum) { @@ -142,9 +148,9 @@ angular.module('quay').directive('triggerSetupGithost', function () { $scope.setLocation = function(location) { $scope.state.currentLocation = location; - $scope.state.isInvalidLocation = false; $scope.trigger['config']['subdir'] = location || ''; $scope.trigger.$ready = true; + checkLocation(); }; $scope.selectRepo = function(repo, org) { @@ -199,6 +205,11 @@ angular.module('quay').directive('triggerSetupGithost', function () { $scope.repoLookahead = repos; }; + $scope.$watch('trigger', function(trigger) { + if (!trigger) { return; } + $scope.supportsFullListing = TriggerService.supportsFullListing(trigger.service) + }); + $scope.$watch('state.currentRepo', function(repo) { if (repo) { $scope.selectRepoInternal(repo); diff --git a/static/js/services/trigger-service.js b/static/js/services/trigger-service.js index 874fc9966..02459adf4 100644 --- a/static/js/services/trigger-service.js +++ b/static/js/services/trigger-service.js @@ -49,7 +49,8 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K } return 'GitHub Repository Push'; - } + }, + 'supports_full_directory_listing': true }, 'bitbucket': { @@ -75,7 +76,8 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K return Features.BITBUCKET_BUILD; }, 'icon': 'fa-bitbucket', - 'title': function() { return 'Bitbucket Repository Push'; } + 'title': function() { return 'Bitbucket Repository Push'; }, + 'supports_full_directory_listing': false }, 'custom-git': { @@ -104,6 +106,15 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K } } + triggerService.supportsFullListing = function(name) { + var type = triggerTypes[name]; + if (!type) { + return false; + } + + return !!type['supports_full_directory_listing']; + }; + triggerService.getTypes = function() { var types = []; for (var key in triggerTypes) {