Work in progress: bitbucket support
This commit is contained in:
Joseph Schorr
parent
c480fb2105
commit
5cc91ed202
11 changed files with 352 additions and 259 deletions
|
|
@ -10,6 +10,7 @@ from github import Github, UnknownObjectException, GithubException
|
|||
from bitbucket.bitbucket import Bitbucket
|
||||
from tempfile import SpooledTemporaryFile
|
||||
from jsonschema import validate
|
||||
from data import model
|
||||
|
||||
from app import app, userfiles as user_files, github_trigger, get_app_url
|
||||
from util.tarfileappender import TarfileAppender
|
||||
|
|
@ -63,59 +64,75 @@ class TriggerProviderException(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class BuildTrigger(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
def raise_unsupported():
|
||||
raise io.UnsupportedOperation
|
||||
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
def get_trigger_config(trigger):
|
||||
try:
|
||||
return json.loads(trigger.config)
|
||||
except:
|
||||
return {}
|
||||
|
||||
|
||||
class BuildTriggerHandler(object):
|
||||
def __init__(self, trigger, override_config=None):
|
||||
self.trigger = trigger
|
||||
self.config = override_config or get_trigger_config(trigger)
|
||||
|
||||
@property
|
||||
def auth_token(self):
|
||||
""" Returns the auth token for the trigger. """
|
||||
return self.trigger.auth_token
|
||||
|
||||
def dockerfile_url(self):
|
||||
"""
|
||||
Returns the URL at which the Dockerfile for the trigger is found or None if none/not applicable.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def load_dockerfile_contents(self, auth_token, config):
|
||||
def load_dockerfile_contents(self):
|
||||
"""
|
||||
Loads the Dockerfile found for the trigger's config and returns them or None if none could
|
||||
be found/loaded.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def list_build_sources(self, auth_token):
|
||||
def list_build_sources(self):
|
||||
"""
|
||||
Take the auth information for the specific trigger type and load the
|
||||
list of build sources(repositories).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def list_build_subdirs(self, auth_token, config):
|
||||
def list_build_subdirs(self):
|
||||
"""
|
||||
Take the auth information and the specified config so far and list all of
|
||||
the possible subdirs containing dockerfiles.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def handle_trigger_request(self, request, trigger):
|
||||
def handle_trigger_request(self):
|
||||
"""
|
||||
Transform the incoming request data into a set of actions. Returns a tuple
|
||||
of usefiles resource id, docker tags, build name, and resource subdir.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def is_active(self, config):
|
||||
def is_active(self):
|
||||
"""
|
||||
Returns True if the current build trigger is active. Inactive means further
|
||||
setup is needed.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
||||
def activate(self, standard_webhook_url):
|
||||
"""
|
||||
Activates the trigger for the service, with the given new configuration.
|
||||
Returns new public and private config that should be stored if successful.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def deactivate(self, auth_token, config):
|
||||
def deactivate(self):
|
||||
"""
|
||||
Deactivates the trigger for the service, removing any hooks installed in
|
||||
the remote service. Returns the new config that should be stored if this
|
||||
|
|
@ -123,13 +140,13 @@ class BuildTrigger(object):
|
|||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def manual_start(self, trigger, run_parameters=None):
|
||||
def manual_start(self, run_parameters=None):
|
||||
"""
|
||||
Manually creates a repository build for this trigger.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def list_field_values(self, auth_token, config, field_name):
|
||||
def list_field_values(self, field_name):
|
||||
"""
|
||||
Lists all values for the given custom trigger field. For example, a trigger might have a
|
||||
field named "branches", and this method would return all branches.
|
||||
|
|
@ -144,25 +161,24 @@ class BuildTrigger(object):
|
|||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def get_trigger_for_service(cls, service):
|
||||
def get_handler(cls, trigger, override_config=None):
|
||||
for subc in cls.__subclasses__():
|
||||
if subc.service_name() == service:
|
||||
return subc()
|
||||
if subc.service_name() == trigger.service.name:
|
||||
return subc(trigger, override_config)
|
||||
|
||||
raise InvalidServiceException('Unable to find service: %s' % service)
|
||||
|
||||
def put_config_key(self, key, value):
|
||||
""" Updates a config key in the trigger, saving it to the DB. """
|
||||
self.config[key] = value
|
||||
model.update_build_trigger(self.trigger, self.config)
|
||||
|
||||
def raise_unsupported():
|
||||
raise io.UnsupportedOperation
|
||||
|
||||
def get_trigger_config(trigger):
|
||||
try:
|
||||
return json.loads(trigger.config)
|
||||
except:
|
||||
return {}
|
||||
def set_auth_token(self, auth_token):
|
||||
""" Sets the auth token for the trigger, saving it to the DB. """
|
||||
model.update_build_trigger(self.trigger, self.config, auth_token=auth_token)
|
||||
|
||||
|
||||
class BitbucketBuildTrigger(BuildTrigger):
|
||||
class BitbucketBuildTrigger(BuildTriggerHandler):
|
||||
"""
|
||||
BuildTrigger for Bitbucket.
|
||||
"""
|
||||
|
|
@ -170,23 +186,25 @@ class BitbucketBuildTrigger(BuildTrigger):
|
|||
def service_name(cls):
|
||||
return 'bitbucket'
|
||||
|
||||
@staticmethod
|
||||
def _get_authorized_client(trigger_uuid):
|
||||
def _get_authorized_client(self, namespace=None):
|
||||
key = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_KEY', '')
|
||||
secret = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_SECRET', '')
|
||||
|
||||
trigger_uuid = self.trigger.uuid
|
||||
callback_url = '%s/oauth1/bitbucket/callback/trigger/%s' % (get_app_url(), trigger_uuid)
|
||||
bitbucket_client = Bitbucket()
|
||||
(result, err_message) = bitbucket_client.authorize(key, secret, callback_url)
|
||||
|
||||
bitbucket_client = Bitbucket(username=namespace or self.config.get('username', ''))
|
||||
|
||||
(result, err_message) = bitbucket_client.authorize(key, secret, callback_url,
|
||||
access_token=self.config.get('access_token'),
|
||||
access_token_secret=self.auth_token)
|
||||
if not result:
|
||||
raise TriggerProviderException(err_message)
|
||||
|
||||
return bitbucket_client
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_oauth_url(trigger_uuid):
|
||||
bitbucket_client = BitbucketBuildTrigger._get_authorized_client(trigger_uuid)
|
||||
def get_oauth_url(self):
|
||||
bitbucket_client = self._get_authorized_client()
|
||||
url = bitbucket_client.url('AUTHENTICATE', token=bitbucket_client.access_token)
|
||||
return {
|
||||
'access_token': bitbucket_client.access_token,
|
||||
|
|
@ -194,62 +212,104 @@ class BitbucketBuildTrigger(BuildTrigger):
|
|||
'url': url
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def exchange_verifier(trigger, verifier):
|
||||
trigger_config = get_trigger_config(trigger.config)
|
||||
bitbucket_client = BitbucketBuildTrigger._get_authorized_client(trigger.uuid)
|
||||
print trigger.config
|
||||
print trigger.auth_token
|
||||
print bitbucket_client.verify(verifier, access_token=trigger_config.get('access_token', ''),
|
||||
access_token_secret=trigger.auth_token)
|
||||
return None
|
||||
#(result, _) = bitbucket_client.verify(verifier)
|
||||
def exchange_verifier(self, verifier):
|
||||
bitbucket_client = self._get_authorized_client()
|
||||
(result, data) = bitbucket_client.verify(verifier,
|
||||
access_token=self.config.get('access_token', ''),
|
||||
access_token_secret=self.auth_token)
|
||||
|
||||
#if not result:
|
||||
# return None
|
||||
if not result:
|
||||
return False
|
||||
|
||||
#return (bitbucket_client.access_token, bitbucket_client.access_token_secret)
|
||||
# Request the user's information and save it and the access token to the config.
|
||||
user_url = bitbucket_client.URLS['BASE'] % 'user'
|
||||
(result, data) = bitbucket_client.dispatch('GET', user_url, auth=bitbucket_client.auth)
|
||||
if not result:
|
||||
return False
|
||||
|
||||
def is_active(self, config):
|
||||
username = data['user']['username']
|
||||
new_access_token = bitbucket_client.access_token
|
||||
|
||||
self.put_config_key('username', username)
|
||||
self.put_config_key('access_token', new_access_token)
|
||||
self.set_auth_token(bitbucket_client.access_token_secret)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_active(self):
|
||||
return False
|
||||
|
||||
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
||||
def activate(self, standard_webhook_url):
|
||||
return {}
|
||||
|
||||
def deactivate(self, auth_token, config):
|
||||
return config
|
||||
def deactivate(self):
|
||||
return self.config
|
||||
|
||||
def list_build_sources(self, auth_token):
|
||||
def list_build_sources(self):
|
||||
bitbucket_client = self._get_authorized_client()
|
||||
success, repositories = bitbucket_client.repository.all()
|
||||
if not success:
|
||||
raise RepositoryReadException('Could not read repository list')
|
||||
|
||||
namespaces = {}
|
||||
|
||||
for repo in repositories:
|
||||
if not repo['scm'] == 'git':
|
||||
continue
|
||||
|
||||
owner = repo['owner']
|
||||
if not owner in namespaces:
|
||||
namespaces[owner] = {
|
||||
'personal': owner == self.config.get('username'),
|
||||
'repos': [],
|
||||
'info': {
|
||||
'name': owner
|
||||
}
|
||||
}
|
||||
|
||||
namespaces[owner]['repos'].append(owner + '/' + repo['slug'])
|
||||
|
||||
return namespaces.values()
|
||||
|
||||
|
||||
def list_build_subdirs(self):
|
||||
source = self.config['build_source']
|
||||
(namespace, name) = source.split('/')
|
||||
(result, data) = self._get_authorized_client(namespace=namespace).repository.get(name)
|
||||
|
||||
print result
|
||||
print data
|
||||
return []
|
||||
|
||||
def dockerfile_url(self):
|
||||
return None
|
||||
|
||||
def load_dockerfile_contents(self):
|
||||
raise RepositoryReadException('Not supported')
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
return
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
return None
|
||||
|
||||
def list_field_values(self, field_name):
|
||||
source = self.config['build_source']
|
||||
(namespace, name) = source.split('/')
|
||||
(result, data) = self._get_authorized_client(namespace=namespace).repository.get(name)
|
||||
|
||||
print result
|
||||
print data
|
||||
return []
|
||||
|
||||
|
||||
def list_build_subdirs(self, auth_token, config):
|
||||
raise RepositoryReadException('Not supported')
|
||||
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
return None
|
||||
|
||||
def load_dockerfile_contents(self, auth_token, config):
|
||||
raise RepositoryReadException('Not supported')
|
||||
|
||||
@staticmethod
|
||||
def _build_commit_info(repo, commit_sha):
|
||||
return {}
|
||||
|
||||
def handle_trigger_request(self, request, trigger):
|
||||
return
|
||||
|
||||
def manual_start(self, trigger, run_parameters=None):
|
||||
return None
|
||||
|
||||
|
||||
class GithubBuildTrigger(BuildTrigger):
|
||||
class GithubBuildTrigger(BuildTriggerHandler):
|
||||
"""
|
||||
BuildTrigger for GitHub that uses the archive API and buildpacks.
|
||||
"""
|
||||
@staticmethod
|
||||
def _get_client(auth_token):
|
||||
return Github(auth_token,
|
||||
def _get_client(self):
|
||||
return Github(self.auth_token,
|
||||
base_url=github_trigger.api_endpoint(),
|
||||
client_id=github_trigger.client_id(),
|
||||
client_secret=github_trigger.client_secret())
|
||||
|
|
@ -258,12 +318,13 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
def service_name(cls):
|
||||
return 'github'
|
||||
|
||||
def is_active(self, config):
|
||||
return 'hook_id' in config
|
||||
def is_active(self):
|
||||
return 'hook_id' in self.config
|
||||
|
||||
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
||||
def activate(self, standard_webhook_url):
|
||||
config = self.config
|
||||
new_build_source = config['build_source']
|
||||
gh_client = self._get_client(auth_token)
|
||||
gh_client = self._get_client()
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
|
|
@ -303,8 +364,9 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
return config, {'private_key': private_key}
|
||||
|
||||
def deactivate(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
def deactivate(self):
|
||||
config = self.config
|
||||
gh_client = self._get_client()
|
||||
|
||||
# Find the GitHub repository.
|
||||
try:
|
||||
|
|
@ -334,11 +396,11 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
raise TriggerDeactivationException(msg)
|
||||
|
||||
config.pop('hook_id', None)
|
||||
|
||||
self.config = config
|
||||
return config
|
||||
|
||||
def list_build_sources(self, auth_token):
|
||||
gh_client = self._get_client(auth_token)
|
||||
def list_build_sources(self):
|
||||
gh_client = self._get_client()
|
||||
usr = gh_client.get_user()
|
||||
|
||||
personal = {
|
||||
|
|
@ -380,8 +442,9 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def list_build_subdirs(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
def list_build_subdirs(self):
|
||||
config = self.config
|
||||
gh_client = self._get_client()
|
||||
source = config['build_source']
|
||||
|
||||
try:
|
||||
|
|
@ -411,11 +474,13 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
raise RepositoryReadException(message)
|
||||
|
||||
def dockerfile_url(self, auth_token, config):
|
||||
def dockerfile_url(self):
|
||||
config = self.config
|
||||
|
||||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
|
||||
gh_client = self._get_client(auth_token)
|
||||
gh_client = self._get_client()
|
||||
|
||||
try:
|
||||
repo = gh_client.get_repo(source)
|
||||
|
|
@ -425,8 +490,9 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
logger.exception('Could not load repository for Dockerfile.')
|
||||
return None
|
||||
|
||||
def load_dockerfile_contents(self, auth_token, config):
|
||||
gh_client = self._get_client(auth_token)
|
||||
def load_dockerfile_contents(self):
|
||||
config = self.config
|
||||
gh_client = self._get_client()
|
||||
|
||||
source = config['build_source']
|
||||
subdirectory = config.get('subdir', '')
|
||||
|
|
@ -556,7 +622,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
def get_display_name(sha):
|
||||
return sha[0:7]
|
||||
|
||||
def handle_trigger_request(self, request, trigger):
|
||||
def handle_trigger_request(self, request):
|
||||
payload = request.get_json()
|
||||
if not payload or payload.get('head_commit') is None:
|
||||
raise SkipRequestException()
|
||||
|
|
@ -570,7 +636,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
commit_message = payload['head_commit'].get('message', '')
|
||||
git_url = payload['repository']['git_url']
|
||||
|
||||
config = get_trigger_config(trigger)
|
||||
config = self.config
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
|
|
@ -585,7 +651,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
short_sha = GithubBuildTrigger.get_display_name(commit_sha)
|
||||
|
||||
gh_client = self._get_client(trigger.auth_token)
|
||||
gh_client = self._get_client()
|
||||
|
||||
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||
payload['repository']['name'])
|
||||
|
|
@ -593,16 +659,16 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
logger.debug('Github repo: %s', repo)
|
||||
|
||||
return GithubBuildTrigger._prepare_build(trigger, config, repo, commit_sha,
|
||||
short_sha, ref, git_url)
|
||||
return GithubBuildTrigger._prepare_build(self.trigger, config, repo, commit_sha, short_sha,
|
||||
ref, git_url)
|
||||
|
||||
def manual_start(self, trigger, run_parameters=None):
|
||||
config = get_trigger_config(trigger)
|
||||
def manual_start(self, run_parameters=None):
|
||||
config = self.config
|
||||
try:
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
||||
gh_client = self._get_client(trigger.auth_token)
|
||||
gh_client = self._get_client()
|
||||
repo = gh_client.get_repo(source)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
|
|
@ -611,27 +677,28 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
ref = 'refs/heads/%s' % (branch_name)
|
||||
git_url = repo.git_url
|
||||
|
||||
return self._prepare_build(trigger, config, repo, branch_sha, short_sha, ref, git_url)
|
||||
return self._prepare_build(self.trigger, config, repo, branch_sha, short_sha, ref, git_url)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
|
||||
def list_field_values(self, auth_token, config, field_name):
|
||||
def list_field_values(self, field_name):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values(auth_token, config, 'branch_name')
|
||||
tags = self.list_field_values(auth_token, config, 'tag_name')
|
||||
branches = self.list_field_values('branch_name')
|
||||
tags = self.list_field_values('tag_name')
|
||||
|
||||
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||
[{'kind': 'tag', 'name': tag} for tag in tags])
|
||||
|
||||
config = self.config
|
||||
if field_name == 'tag_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
gh_client = self._get_client()
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
return [tag.name for tag in repo.get_tags()]
|
||||
|
||||
if field_name == 'branch_name':
|
||||
gh_client = self._get_client(auth_token)
|
||||
gh_client = self._get_client()
|
||||
source = config['build_source']
|
||||
repo = gh_client.get_repo(source)
|
||||
branches = [branch.name for branch in repo.get_branches()]
|
||||
|
|
@ -647,7 +714,7 @@ class GithubBuildTrigger(BuildTrigger):
|
|||
|
||||
return None
|
||||
|
||||
class CustomBuildTrigger(BuildTrigger):
|
||||
class CustomBuildTrigger(BuildTriggerHandler):
|
||||
payload_schema = {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
|
|
@ -730,8 +797,8 @@ class CustomBuildTrigger(BuildTrigger):
|
|||
def service_name(cls):
|
||||
return 'custom-git'
|
||||
|
||||
def is_active(self, config):
|
||||
return config.has_key('credentials')
|
||||
def is_active(self):
|
||||
return self.config.has_key('credentials')
|
||||
|
||||
def _metadata_from_payload(self, payload):
|
||||
try:
|
||||
|
|
@ -741,7 +808,7 @@ class CustomBuildTrigger(BuildTrigger):
|
|||
raise InvalidPayloadException()
|
||||
return metadata
|
||||
|
||||
def handle_trigger_request(self, request, trigger):
|
||||
def handle_trigger_request(self, request):
|
||||
payload = request.get_json()
|
||||
if not payload:
|
||||
raise SkipRequestException()
|
||||
|
|
@ -750,7 +817,7 @@ class CustomBuildTrigger(BuildTrigger):
|
|||
metadata = self._metadata_from_payload(payload)
|
||||
|
||||
# The build source is the canonical git URL used to clone.
|
||||
config = get_trigger_config(trigger)
|
||||
config = self.config
|
||||
metadata['git_url'] = config['build_source']
|
||||
|
||||
branch = metadata['ref'].split('/')[-1]
|
||||
|
|
@ -759,9 +826,10 @@ class CustomBuildTrigger(BuildTrigger):
|
|||
build_name = metadata['commit_sha'][:6]
|
||||
dockerfile_id = None
|
||||
|
||||
return dockerfile_id, tags, build_name, trigger.config['subdir'], metadata
|
||||
return dockerfile_id, tags, build_name, config['subdir'], metadata
|
||||
|
||||
def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
|
||||
def activate(self, standard_webhook_url):
|
||||
config = self.config
|
||||
public_key, private_key = generate_ssh_keypair()
|
||||
config['credentials'] = [
|
||||
{
|
||||
|
|
@ -773,18 +841,21 @@ class CustomBuildTrigger(BuildTrigger):
|
|||
'value': standard_webhook_url,
|
||||
},
|
||||
]
|
||||
self.config = config
|
||||
return config, {'private_key': private_key}
|
||||
|
||||
def deactivate(self, auth_token, config):
|
||||
def deactivate(self):
|
||||
config = self.config
|
||||
config.pop('credentials', None)
|
||||
self.config = config
|
||||
return config
|
||||
|
||||
def manual_start(self, trigger, run_parameters=None):
|
||||
def manual_start(self, run_parameters=None):
|
||||
# commit_sha is the only required parameter
|
||||
if 'commit_sha' not in run_parameters:
|
||||
raise TriggerStartException('missing required parameter')
|
||||
|
||||
config = get_trigger_config(trigger)
|
||||
config = self.config
|
||||
dockerfile_id = None
|
||||
tags = {run_parameters['commit_sha']}
|
||||
build_name = run_parameters['commit_sha']
|
||||
|
|
|
|||
Reference in a new issue