Get build preparation working for bitbucket and do a lot of code cleanup around this process across all the triggers. Note: tests are not yet updated.
This commit is contained in:
Joseph Schorr
parent
6479f8ddc9
commit
d5c70878c5
6 changed files with 432 additions and 226 deletions
|
|
@ -6,6 +6,7 @@ import base64
|
|||
import re
|
||||
import json
|
||||
|
||||
from endpoints.building import PreparedBuild
|
||||
from github import Github, UnknownObjectException, GithubException
|
||||
from bitbucket import BitBucket
|
||||
from tempfile import SpooledTemporaryFile
|
||||
|
|
@ -27,9 +28,6 @@ TARBALL_MIME = 'application/gzip'
|
|||
CHUNK_SIZE = 512 * 1024
|
||||
|
||||
|
||||
def should_skip_commit(message):
|
||||
return '[skip build]' in message or '[build skip]' in message
|
||||
|
||||
class InvalidPayloadException(Exception):
|
||||
pass
|
||||
|
||||
|
|
@ -64,6 +62,42 @@ class TriggerProviderException(Exception):
|
|||
pass
|
||||
|
||||
|
||||
def find_matching_branches(config, branches):
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
return [branch for branch in branches
|
||||
if matches_ref('refs/heads/' + branch, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
return branches
|
||||
|
||||
def raise_if_skipped(config, ref):
|
||||
""" Raises a SkipRequestException if the given ref should be skipped. """
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
def matches_ref(ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
||||
m = regex.match(match_string)
|
||||
if not m:
|
||||
return False
|
||||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def should_skip_commit(message):
|
||||
return '[skip build]' in message or '[build skip]' in message
|
||||
|
||||
def raise_unsupported():
|
||||
raise io.UnsupportedOperation
|
||||
|
||||
|
|
@ -113,8 +147,7 @@ class BuildTriggerHandler(object):
|
|||
|
||||
def handle_trigger_request(self):
|
||||
"""
|
||||
Transform the incoming request data into a set of actions. Returns a tuple
|
||||
of usefiles resource id, docker tags, build name, and resource subdir.
|
||||
Transform the incoming request data into a set of actions. Returns a PreparedBuild.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -142,7 +175,7 @@ class BuildTriggerHandler(object):
|
|||
|
||||
def manual_start(self, run_parameters=None):
|
||||
"""
|
||||
Manually creates a repository build for this trigger.
|
||||
Manually creates a repository build for this trigger. Returns a PreparedBuild.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -166,7 +199,7 @@ class BuildTriggerHandler(object):
|
|||
if subc.service_name() == trigger.service.name:
|
||||
return subc(trigger, override_config)
|
||||
|
||||
raise InvalidServiceException('Unable to find service: %s' % service)
|
||||
raise InvalidServiceException('Unable to find service: %s' % trigger.service.name)
|
||||
|
||||
def put_config_key(self, key, value):
|
||||
""" Updates a config key in the trigger, saving it to the DB. """
|
||||
|
|
@ -272,7 +305,6 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
config['hook_id'] = data['id']
|
||||
return config, {'private_key': private_key}
|
||||
|
||||
|
||||
def deactivate(self):
|
||||
config = self.config
|
||||
repository = self._get_repository_client()
|
||||
|
|
@ -294,7 +326,6 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
return config
|
||||
|
||||
|
||||
def list_build_sources(self):
|
||||
bitbucket_client = self._get_authorized_client()
|
||||
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
|
||||
|
|
@ -321,12 +352,16 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
return namespaces.values()
|
||||
|
||||
def list_build_subdirs(self):
|
||||
config = self.config
|
||||
repository = self._get_repository_client()
|
||||
(result, data, err_msg) = repository.get_path_contents('', revision='master')
|
||||
|
||||
# Find the first matching branch.
|
||||
repo_branches = self.list_field_values('branch_name') or []
|
||||
branches = find_matching_branches(config, repo_branches)
|
||||
(result, data, err_msg) = repository.get_path_contents('', revision=branches[0])
|
||||
if not result:
|
||||
raise RepositoryReadException(err_msg)
|
||||
|
||||
|
||||
files = set([f['path'] for f in data['files']])
|
||||
if 'Dockerfile' in files:
|
||||
return ['/']
|
||||
|
|
@ -392,29 +427,114 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
return None
|
||||
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
return
|
||||
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
def _prepare_build(self, commit_sha, ref, is_manual):
|
||||
config = self.config
|
||||
repository = self._get_repository_client()
|
||||
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
||||
# Lookup the branch to build.
|
||||
master_branch = 'master'
|
||||
# Lookup the default branch associated with the repository. We use this when building
|
||||
# the tags.
|
||||
default_branch = ''
|
||||
(result, data, _) = repository.get_main_branch()
|
||||
if result:
|
||||
master_branch = data['name']
|
||||
default_branch = data['name']
|
||||
|
||||
branch_name = run_parameters.get('branch_name') or master_branch
|
||||
# Lookup the commit sha.
|
||||
(result, data, _) = repository.changesets().get(commit_sha)
|
||||
if not result:
|
||||
raise TriggerStartException('Could not lookup commit SHA')
|
||||
|
||||
# Find the SHA for the branch.
|
||||
# TODO
|
||||
return None
|
||||
namespace = repository.namespace
|
||||
name = repository.repository_name
|
||||
|
||||
commit_info = {
|
||||
'url': 'https://bitbucket.org/%s/%s/commits/%s' % (namespace, name, commit_sha),
|
||||
'message': data['message'],
|
||||
'date': data['timestamp']
|
||||
}
|
||||
|
||||
# Try to lookup the author by email address. The raw_author field (if it exists) is returned
|
||||
# in the form: "Joseph Schorr <joseph.schorr@coreos.com>"
|
||||
if data.get('raw_author'):
|
||||
match = re.compile(r'.*<(.+)>').match(data['raw_author'])
|
||||
if match:
|
||||
email_address = match.group(1)
|
||||
bitbucket_client = self._get_authorized_client()
|
||||
(result, data, _) = bitbucket_client.accounts().get_profile(email_address)
|
||||
if result:
|
||||
commit_info['author'] = {
|
||||
'username': data['user']['username'],
|
||||
'url': 'https://bitbucket.org/%s/' % data['user']['username'],
|
||||
'avatar_url': data['user']['avatar']
|
||||
}
|
||||
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'ref': ref,
|
||||
'default_branch': default_branch,
|
||||
'git_url': 'git@bitbucket.org:%s/%s.git' % (namespace, name),
|
||||
'commit_info': commit_info
|
||||
}
|
||||
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
prepared.tags_from_ref(ref, default_branch)
|
||||
prepared.name_from_sha(commit_sha)
|
||||
prepared.subdirectory = config['subdir']
|
||||
prepared.metadata = metadata
|
||||
prepared.is_manual = is_manual
|
||||
|
||||
return prepared
|
||||
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
# Parse the JSON payload.
|
||||
payload_json = request.form.get('payload')
|
||||
if not payload_json:
|
||||
raise SkipRequestException()
|
||||
|
||||
try:
|
||||
payload = json.loads(payload_json)
|
||||
except ValueError:
|
||||
raise SkipRequestException()
|
||||
|
||||
logger.debug('BitBucket trigger payload %s', payload)
|
||||
|
||||
# Make sure we have a commit in the payload.
|
||||
if not payload.get('commits'):
|
||||
raise SkipRequestException()
|
||||
|
||||
# Check if this build should be skipped by commit message.
|
||||
commit = payload['commits'][0]
|
||||
commit_message = commit['message']
|
||||
if should_skip_commit(commit_message):
|
||||
raise SkipRequestException()
|
||||
|
||||
# Check to see if this build should be skipped by ref.
|
||||
ref = 'refs/heads/' + commit['branch'] if commit.get('branch') else 'refs/tags/' + commit['tag']
|
||||
raise_if_skipped(self.config, ref)
|
||||
|
||||
commit_sha = commit['node']
|
||||
return self._prepare_build(commit_sha, ref, False)
|
||||
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
run_parameters = run_parameters or {}
|
||||
repository = self._get_repository_client()
|
||||
|
||||
# Find the branch to build.
|
||||
branch_name = run_parameters.get('branch_name')
|
||||
(result, data, _) = repository.get_main_branch()
|
||||
if result:
|
||||
branch_name = data['name'] or branch_name
|
||||
|
||||
# Lookup the commit SHA for the branch.
|
||||
(result, data, _) = repository.get_branches()
|
||||
if not result or not branch_name in data:
|
||||
raise TriggerStartException('Could not find branch commit SHA')
|
||||
|
||||
commit_sha = data[branch_name]['node']
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
return self._prepare_build(commit_sha, ref, True)
|
||||
|
||||
|
||||
class GithubBuildTrigger(BuildTriggerHandler):
|
||||
|
|
@ -543,18 +663,6 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
return repos_by_org
|
||||
|
||||
@staticmethod
|
||||
def matches_ref(ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
||||
m = regex.match(match_string)
|
||||
if not m:
|
||||
return False
|
||||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def list_build_subdirs(self):
|
||||
config = self.config
|
||||
gh_client = self._get_client()
|
||||
|
|
@ -564,15 +672,8 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
repo = gh_client.get_repo(source)
|
||||
|
||||
# Find the first matching branch.
|
||||
branches = None
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
branches = [branch.name for branch in repo.get_branches()
|
||||
if GithubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
repo_branches = self.list_field_values('branch_name') or []
|
||||
branches = find_matching_branches(config, repo_branches)
|
||||
branches = branches or [repo.default_branch or 'master']
|
||||
default_commit = repo.get_branch(branches[0]).commit
|
||||
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
||||
|
|
@ -691,37 +792,31 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
return tarball_subdir, dockerfile_id
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _prepare_build(trigger, config, repo, commit_sha, build_name, ref, git_url):
|
||||
repo_subdir = config['subdir']
|
||||
joined_subdir = repo_subdir
|
||||
dockerfile_id = None
|
||||
def _prepare_build(self, repo, ref, commit_sha, is_manual):
|
||||
config = self.config
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
|
||||
if trigger.private_key is None:
|
||||
# If the trigger isn't using git, prepare the buildpack.
|
||||
# If the trigger isn't using git, prepare the buildpack.
|
||||
if self.trigger.private_key is None:
|
||||
tarball_subdir, dockerfile_id = GithubBuildTrigger._prepare_tarball(repo, commit_sha)
|
||||
logger.debug('Successfully prepared job')
|
||||
|
||||
# Join provided subdir with the tarball subdir.
|
||||
joined_subdir = os.path.join(tarball_subdir, repo_subdir)
|
||||
prepared.subdirectory = os.path.join(tarball_subdir, config['subdir'])
|
||||
prepared.dockerfile_id = dockerfile_id
|
||||
else:
|
||||
prepared.subdirectory = config['subdir']
|
||||
|
||||
logger.debug('Final subdir: %s', joined_subdir)
|
||||
# Set the name.
|
||||
prepared.name_from_sha(commit_sha)
|
||||
|
||||
# compute the tag(s)
|
||||
branch = ref.split('/')[-1]
|
||||
tags = {branch}
|
||||
# Set the tag(s).
|
||||
prepared.tags_from_ref(ref, repo.default_branch)
|
||||
|
||||
if branch == repo.default_branch:
|
||||
tags.add('latest')
|
||||
|
||||
logger.debug('Pushing to tags: %s', tags)
|
||||
|
||||
# compute the metadata
|
||||
# Build and set the metadata.
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'ref': ref,
|
||||
'default_branch': repo.default_branch,
|
||||
'git_url': git_url,
|
||||
'git_url': repo.git_url,
|
||||
}
|
||||
|
||||
# add the commit info.
|
||||
|
|
@ -729,71 +824,63 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
if commit_info is not None:
|
||||
metadata['commit_info'] = commit_info
|
||||
|
||||
return dockerfile_id, list(tags), build_name, joined_subdir, metadata
|
||||
prepared.metadata = metadata
|
||||
prepared.is_manual = is_manual
|
||||
return prepared
|
||||
|
||||
@staticmethod
|
||||
def get_display_name(sha):
|
||||
return sha[0:7]
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
# Check the payload to see if we should skip it based on the lack of a head_commit.
|
||||
payload = request.get_json()
|
||||
if not payload or payload.get('head_commit') is None:
|
||||
raise SkipRequestException()
|
||||
|
||||
# This is for GitHub's probing/testing.
|
||||
if 'zen' in payload:
|
||||
raise ValidationRequestException()
|
||||
|
||||
logger.debug('Payload %s', payload)
|
||||
logger.debug('GitHub trigger payload %s', payload)
|
||||
|
||||
ref = payload['ref']
|
||||
commit_sha = payload['head_commit']['id']
|
||||
commit_message = payload['head_commit'].get('message', '')
|
||||
git_url = payload['repository']['git_url']
|
||||
|
||||
config = self.config
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not GithubBuildTrigger.matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
# Check if this build should be skipped by commit message.
|
||||
if should_skip_commit(commit_message):
|
||||
raise SkipRequestException()
|
||||
|
||||
short_sha = GithubBuildTrigger.get_display_name(commit_sha)
|
||||
# Check to see if this build should be skipped by ref.
|
||||
raise_if_skipped(self.config, ref)
|
||||
|
||||
gh_client = self._get_client()
|
||||
|
||||
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||
payload['repository']['name'])
|
||||
repo = gh_client.get_repo(repo_full_name)
|
||||
|
||||
logger.debug('Github repo: %s', repo)
|
||||
|
||||
return GithubBuildTrigger._prepare_build(self.trigger, config, repo, commit_sha, short_sha,
|
||||
ref, git_url)
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
config = self.config
|
||||
try:
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||
payload['repository']['name'])
|
||||
|
||||
gh_client = self._get_client()
|
||||
repo = gh_client.get_repo(source)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
branch_sha = branch.commit.sha
|
||||
short_sha = GithubBuildTrigger.get_display_name(branch_sha)
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
git_url = repo.git_url
|
||||
repo = gh_client.get_repo(repo_full_name)
|
||||
|
||||
return self._prepare_build(self.trigger, config, repo, branch_sha, short_sha, ref, git_url)
|
||||
return self._prepare_build(repo, ref, commit_sha, False)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
config = self.config
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
||||
try:
|
||||
gh_client = self._get_client()
|
||||
|
||||
# Lookup the branch and its associated current SHA.
|
||||
repo = gh_client.get_repo(source)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
commit_sha = branch.commit.sha
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
return self._prepare_build(repo, ref, commit_sha, True)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
def list_field_values(self, field_name):
|
||||
if field_name == 'refs':
|
||||
|
|
@ -922,24 +1009,50 @@ class CustomBuildTrigger(BuildTriggerHandler):
|
|||
return metadata
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
# Skip if there is no payload.
|
||||
payload = request.get_json()
|
||||
if not payload:
|
||||
raise SkipRequestException()
|
||||
|
||||
logger.debug('Payload %s', payload)
|
||||
|
||||
# Skip if the commit message matches.
|
||||
metadata = self._metadata_from_payload(payload)
|
||||
if should_skip_commit(metadata.get('commit_info', {}).get('message', '')):
|
||||
raise SkipRequestException()
|
||||
|
||||
# The build source is the canonical git URL used to clone.
|
||||
config = self.config
|
||||
metadata['git_url'] = config['build_source']
|
||||
|
||||
branch = metadata['ref'].split('/')[-1]
|
||||
tags = {branch}
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
prepared.tags_from_ref(metadata['ref'])
|
||||
prepared.name_from_sha(metadata['commit_sha'])
|
||||
prepared.subdirectory = config['subdir']
|
||||
prepared.metadata = metadata
|
||||
|
||||
build_name = metadata['commit_sha'][:6]
|
||||
dockerfile_id = None
|
||||
return prepared
|
||||
|
||||
return dockerfile_id, tags, build_name, config['subdir'], metadata
|
||||
def manual_start(self, run_parameters=None):
|
||||
# commit_sha is the only required parameter
|
||||
commit_sha = run_parameters.get('commit_sha')
|
||||
if commit_sha is None:
|
||||
raise TriggerStartException('missing required parameter')
|
||||
|
||||
config = self.config
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'git_url': config['build_source'],
|
||||
}
|
||||
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
prepared.tags = [commit_sha]
|
||||
prepared.name_from_sha(commit_sha)
|
||||
prepared.subdirectory = config['subdir']
|
||||
prepared.metadata = metadata
|
||||
prepared.is_manual = True
|
||||
|
||||
return prepared
|
||||
|
||||
def activate(self, standard_webhook_url):
|
||||
config = self.config
|
||||
|
|
@ -962,19 +1075,3 @@ class CustomBuildTrigger(BuildTriggerHandler):
|
|||
config.pop('credentials', None)
|
||||
self.config = config
|
||||
return config
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
# commit_sha is the only required parameter
|
||||
if 'commit_sha' not in run_parameters:
|
||||
raise TriggerStartException('missing required parameter')
|
||||
|
||||
config = self.config
|
||||
dockerfile_id = None
|
||||
tags = {run_parameters['commit_sha']}
|
||||
build_name = run_parameters['commit_sha']
|
||||
metadata = {
|
||||
'commit_sha': run_parameters['commit_sha'],
|
||||
'git_url': config['build_source'],
|
||||
}
|
||||
|
||||
return dockerfile_id, list(tags), build_name, config['subdir'], metadata
|
||||
|
|
|
|||
Reference in a new issue