Get build preparation working for bitbucket and do a lot of code cleanup around this process across all the triggers. Note: tests are not yet updated.
This commit is contained in:
Joseph Schorr
parent
6479f8ddc9
commit
d5c70878c5
6 changed files with 432 additions and 226 deletions
|
|
@ -10,7 +10,7 @@ from endpoints.api import (RepositoryParamResource, parse_args, query_param, nic
|
|||
require_repo_read, require_repo_write, validate_json_request,
|
||||
ApiResource, internal_only, format_date, api, Unauthorized, NotFound,
|
||||
path_param, InvalidRequest, require_repo_admin)
|
||||
from endpoints.common import start_build
|
||||
from endpoints.building import start_build, PreparedBuild
|
||||
from endpoints.trigger import BuildTriggerHandler
|
||||
from data import model, database
|
||||
from auth.auth_context import get_authenticated_user
|
||||
|
|
@ -191,8 +191,8 @@ class RepositoryBuildList(RepositoryParamResource):
|
|||
raise Unauthorized()
|
||||
|
||||
# Check if the dockerfile resource has already been used. If so, then it
|
||||
# can only be reused if the user has access to the repository for which it
|
||||
# was used.
|
||||
# can only be reused if the user has access to the repository in which the
|
||||
# dockerfile was previously built.
|
||||
associated_repository = model.get_repository_for_resource(dockerfile_id)
|
||||
if associated_repository:
|
||||
if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
|
||||
|
|
@ -201,11 +201,16 @@ class RepositoryBuildList(RepositoryParamResource):
|
|||
|
||||
# Start the build.
|
||||
repo = model.get_repository(namespace, repository)
|
||||
display_name = user_files.get_file_checksum(dockerfile_id)
|
||||
|
||||
build_request = start_build(repo, dockerfile_id, tags, display_name, subdir, True,
|
||||
pull_robot_name=pull_robot_name)
|
||||
prepared = PreparedBuild()
|
||||
prepared.build_name = user_files.get_file_checksum(dockerfile_id)
|
||||
prepared.dockerfile_id = dockerfile_id
|
||||
prepared.tags = tags
|
||||
prepared.subdirectory = subdir
|
||||
prepared.is_manual = True
|
||||
prepared.metadata = {}
|
||||
|
||||
build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
|
||||
resp = build_status_view(build_request, can_write=True)
|
||||
repo_string = '%s/%s' % (namespace, repository)
|
||||
headers = {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from endpoints.api import (RepositoryParamResource, nickname, resource, require_
|
|||
path_param)
|
||||
from endpoints.api.build import (build_status_view, trigger_view, RepositoryBuildStatus,
|
||||
get_trigger_config)
|
||||
from endpoints.common import start_build
|
||||
from endpoints.building import start_build
|
||||
from endpoints.trigger import (BuildTriggerHandler, TriggerDeactivationException,
|
||||
TriggerActivationException, EmptyRepositoryException,
|
||||
RepositoryReadException, TriggerStartException)
|
||||
|
|
@ -423,16 +423,12 @@ class ActivateBuildTrigger(RepositoryParamResource):
|
|||
raise InvalidRequest('Trigger is not active.')
|
||||
|
||||
try:
|
||||
run_parameters = request.get_json()
|
||||
specs = handler.manual_start(run_parameters=run_parameters)
|
||||
dockerfile_id, tags, name, subdir, metadata = specs
|
||||
|
||||
repo = model.get_repository(namespace, repository)
|
||||
pull_robot_name = model.get_pull_robot_name(trigger)
|
||||
|
||||
build_request = start_build(repo, dockerfile_id, tags, name, subdir, True,
|
||||
trigger=trigger, pull_robot_name=pull_robot_name,
|
||||
trigger_metadata=metadata)
|
||||
run_parameters = request.get_json()
|
||||
prepared = handler.manual_start(run_parameters=run_parameters)
|
||||
build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
|
||||
except TriggerStartException as tse:
|
||||
raise InvalidRequest(tse.message)
|
||||
|
||||
|
|
|
|||
186
endpoints/building.py
Normal file
186
endpoints/building.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
import logging
|
||||
import json
|
||||
|
||||
from app import app, dockerfile_build_queue
|
||||
from data import model
|
||||
from data.database import db
|
||||
from auth.auth_context import get_authenticated_user
|
||||
from endpoints.notificationhelper import spawn_notification
|
||||
from flask import request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def start_build(repository, prepared_build, pull_robot_name=None):
|
||||
host = app.config['SERVER_HOSTNAME']
|
||||
repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)
|
||||
|
||||
token = model.create_access_token(repository, 'write', kind='build-worker',
|
||||
friendly_name='Repository Build Token')
|
||||
logger.debug('Creating build %s with repo %s tags %s',
|
||||
prepared_build.build_name, repo_path, prepared_build.tags)
|
||||
|
||||
job_config = {
|
||||
'docker_tags': prepared_build.tags,
|
||||
'registry': host,
|
||||
'build_subdir': prepared_build.subdirectory,
|
||||
'trigger_metadata': prepared_build.metadata or {},
|
||||
'is_manual': prepared_build.is_manual,
|
||||
'manual_user': get_authenticated_user().username if get_authenticated_user() else None
|
||||
}
|
||||
|
||||
with app.config['DB_TRANSACTION_FACTORY'](db):
|
||||
build_request = model.create_repository_build(repository, token, job_config,
|
||||
prepared_build.dockerfile_id,
|
||||
prepared_build.build_name,
|
||||
prepared_build.trigger,
|
||||
pull_robot_name=pull_robot_name)
|
||||
|
||||
json_data = json.dumps({
|
||||
'build_uuid': build_request.uuid,
|
||||
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
|
||||
})
|
||||
|
||||
queue_id = dockerfile_build_queue.put([repository.namespace_user.username, repository.name],
|
||||
json_data,
|
||||
retries_remaining=3)
|
||||
|
||||
build_request.queue_id = queue_id
|
||||
build_request.save()
|
||||
|
||||
# Add the build to the repo's log and spawn the build_queued notification.
|
||||
event_log_metadata = {
|
||||
'build_uuid': build_request.uuid,
|
||||
'docker_tags': prepared_build.tags,
|
||||
'repo': repository.name,
|
||||
'namespace': repository.namespace_user.username,
|
||||
'is_manual': prepared_build.is_manual,
|
||||
'manual_user': get_authenticated_user().username if get_authenticated_user() else None
|
||||
}
|
||||
|
||||
if prepared_build.trigger:
|
||||
event_log_metadata['trigger_id'] = prepared_build.trigger.uuid
|
||||
event_log_metadata['trigger_kind'] = prepared_build.trigger.service.name
|
||||
|
||||
model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr,
|
||||
metadata=event_log_metadata, repository=repository)
|
||||
|
||||
spawn_notification(repository, 'build_queued', event_log_metadata,
|
||||
subpage='build?current=%s' % build_request.uuid,
|
||||
pathargs=['build', build_request.uuid])
|
||||
|
||||
return build_request
|
||||
|
||||
|
||||
class PreparedBuild(object):
|
||||
""" Class which holds all the information about a prepared build. The build queuing service
|
||||
will use this result to actually invoke the build.
|
||||
"""
|
||||
def __init__(self, trigger=None):
|
||||
self._dockerfile_id = None
|
||||
self._tags = None
|
||||
self._build_name = None
|
||||
self._subdirectory = None
|
||||
self._metadata = None
|
||||
self._trigger = trigger
|
||||
self._is_manual = None
|
||||
|
||||
@staticmethod
|
||||
def get_display_name(sha):
|
||||
return sha[0:7]
|
||||
|
||||
def tags_from_ref(self, ref, default_branch='master'):
|
||||
branch = ref.split('/')[-1]
|
||||
tags = {branch}
|
||||
|
||||
if branch == default_branch:
|
||||
tags.add('latest')
|
||||
|
||||
self.tags = tags
|
||||
|
||||
def name_from_sha(self, sha):
|
||||
self.build_name = PreparedBuild.get_display_name(sha)
|
||||
|
||||
@property
|
||||
def is_manual(self):
|
||||
if self._is_manual is None:
|
||||
raise Exception('Property is_manual not set')
|
||||
|
||||
return self._is_manual
|
||||
|
||||
@is_manual.setter
|
||||
def is_manual(self, value):
|
||||
if self._is_manual is not None:
|
||||
raise Exception('Property is_manual already set')
|
||||
|
||||
self._is_manual = value
|
||||
|
||||
@property
|
||||
def trigger(self):
|
||||
return self._trigger
|
||||
|
||||
@property
|
||||
def dockerfile_id(self):
|
||||
return self._dockerfile_id
|
||||
|
||||
@dockerfile_id.setter
|
||||
def dockerfile_id(self, value):
|
||||
if self._dockerfile_id:
|
||||
raise Exception('Property dockerfile_id already set')
|
||||
|
||||
self._dockerfile_id = value
|
||||
|
||||
@property
|
||||
def tags(self):
|
||||
if not self._tags:
|
||||
raise Exception('Missing property tags')
|
||||
|
||||
return self._tags
|
||||
|
||||
@tags.setter
|
||||
def tags(self, value):
|
||||
if self._tags:
|
||||
raise Exception('Property tags already set')
|
||||
|
||||
self._tags = list(value)
|
||||
|
||||
@property
|
||||
def build_name(self):
|
||||
if not self._build_name:
|
||||
raise Exception('Missing property build_name')
|
||||
|
||||
return self._build_name
|
||||
|
||||
@build_name.setter
|
||||
def build_name(self, value):
|
||||
if self._build_name:
|
||||
raise Exception('Property build_name already set')
|
||||
|
||||
self._build_name = value
|
||||
|
||||
@property
|
||||
def subdirectory(self):
|
||||
if self._subdirectory is None:
|
||||
raise Exception('Missing property subdirectory')
|
||||
|
||||
return self._subdirectory
|
||||
|
||||
@subdirectory.setter
|
||||
def subdirectory(self, value):
|
||||
if self._subdirectory:
|
||||
raise Exception('Property subdirectory already set')
|
||||
|
||||
self._subdirectory = value
|
||||
|
||||
@property
|
||||
def metadata(self):
|
||||
if self._metadata is None:
|
||||
raise Exception('Missing property metadata')
|
||||
|
||||
return self._metadata
|
||||
|
||||
@metadata.setter
|
||||
def metadata(self, value):
|
||||
if self._metadata:
|
||||
raise Exception('Property metadata already set')
|
||||
|
||||
self._metadata = value
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import logging
|
||||
import urlparse
|
||||
import json
|
||||
import string
|
||||
import datetime
|
||||
|
|
@ -14,18 +13,15 @@ from flask.ext.principal import identity_changed
|
|||
from random import SystemRandom
|
||||
|
||||
from data import model
|
||||
from data.database import db
|
||||
from app import app, oauth_apps, dockerfile_build_queue, LoginWrappedDBUser
|
||||
from app import app, oauth_apps, LoginWrappedDBUser
|
||||
|
||||
from auth.permissions import QuayDeferredPermissionUser
|
||||
from auth import scopes
|
||||
from auth.auth_context import get_authenticated_user
|
||||
from endpoints.api.discovery import swagger_route_data
|
||||
from werkzeug.routing import BaseConverter
|
||||
from functools import wraps
|
||||
from config import getFrontendVisibleConfig
|
||||
from external_libraries import get_external_javascript, get_external_css
|
||||
from endpoints.notificationhelper import spawn_notification
|
||||
|
||||
import features
|
||||
|
||||
|
|
@ -210,75 +206,3 @@ def check_repository_usage(user_or_org, plan_found):
|
|||
else:
|
||||
model.delete_notifications_by_kind(user_or_org, 'over_private_usage')
|
||||
|
||||
|
||||
def start_build(repository, dockerfile_id, tags, build_name, subdir, manual,
|
||||
trigger=None, pull_robot_name=None, trigger_metadata=None):
|
||||
host = urlparse.urlparse(request.url).netloc
|
||||
repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name)
|
||||
|
||||
token = model.create_access_token(repository, 'write', kind='build-worker',
|
||||
friendly_name='Repository Build Token')
|
||||
logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
|
||||
build_name, repo_path, tags, dockerfile_id)
|
||||
|
||||
job_config = {
|
||||
'docker_tags': tags,
|
||||
'registry': host,
|
||||
'build_subdir': subdir,
|
||||
'trigger_metadata': trigger_metadata or {},
|
||||
'is_manual': manual,
|
||||
'manual_user': get_authenticated_user().username if get_authenticated_user() else None
|
||||
}
|
||||
|
||||
with app.config['DB_TRANSACTION_FACTORY'](db):
|
||||
build_request = model.create_repository_build(repository, token, job_config,
|
||||
dockerfile_id, build_name,
|
||||
trigger, pull_robot_name=pull_robot_name)
|
||||
|
||||
json_data = json.dumps({
|
||||
'build_uuid': build_request.uuid,
|
||||
'pull_credentials': model.get_pull_credentials(pull_robot_name) if pull_robot_name else None
|
||||
})
|
||||
|
||||
queue_id = dockerfile_build_queue.put([repository.namespace_user.username, repository.name],
|
||||
json_data,
|
||||
retries_remaining=3)
|
||||
|
||||
build_request.queue_id = queue_id
|
||||
build_request.save()
|
||||
|
||||
# Add the build to the repo's log.
|
||||
metadata = {
|
||||
'repo': repository.name,
|
||||
'namespace': repository.namespace_user.username,
|
||||
'fileid': dockerfile_id,
|
||||
'is_manual': manual,
|
||||
'manual_user': get_authenticated_user().username if get_authenticated_user() else None
|
||||
}
|
||||
|
||||
if trigger:
|
||||
metadata['trigger_id'] = trigger.uuid
|
||||
metadata['config'] = json.loads(trigger.config)
|
||||
metadata['service'] = trigger.service.name
|
||||
|
||||
model.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr,
|
||||
metadata=metadata, repository=repository)
|
||||
|
||||
# Add notifications for the build queue.
|
||||
logger.debug('Adding notifications for repository')
|
||||
event_data = {
|
||||
'build_id': build_request.uuid,
|
||||
'build_name': build_name,
|
||||
'docker_tags': tags,
|
||||
'is_manual': manual,
|
||||
'manual_user': get_authenticated_user().username if get_authenticated_user() else None
|
||||
}
|
||||
|
||||
if trigger:
|
||||
event_data['trigger_id'] = trigger.uuid
|
||||
event_data['trigger_kind'] = trigger.service.name
|
||||
|
||||
spawn_notification(repository, 'build_queued', event_data,
|
||||
subpage='build?current=%s' % build_request.uuid,
|
||||
pathargs=['build', build_request.uuid])
|
||||
return build_request
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import base64
|
|||
import re
|
||||
import json
|
||||
|
||||
from endpoints.building import PreparedBuild
|
||||
from github import Github, UnknownObjectException, GithubException
|
||||
from bitbucket import BitBucket
|
||||
from tempfile import SpooledTemporaryFile
|
||||
|
|
@ -27,9 +28,6 @@ TARBALL_MIME = 'application/gzip'
|
|||
CHUNK_SIZE = 512 * 1024
|
||||
|
||||
|
||||
def should_skip_commit(message):
|
||||
return '[skip build]' in message or '[build skip]' in message
|
||||
|
||||
class InvalidPayloadException(Exception):
|
||||
pass
|
||||
|
||||
|
|
@ -64,6 +62,42 @@ class TriggerProviderException(Exception):
|
|||
pass
|
||||
|
||||
|
||||
def find_matching_branches(config, branches):
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
return [branch for branch in branches
|
||||
if matches_ref('refs/heads/' + branch, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
return branches
|
||||
|
||||
def raise_if_skipped(config, ref):
|
||||
""" Raises a SkipRequestException if the given ref should be skipped. """
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
def matches_ref(ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
||||
m = regex.match(match_string)
|
||||
if not m:
|
||||
return False
|
||||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def should_skip_commit(message):
|
||||
return '[skip build]' in message or '[build skip]' in message
|
||||
|
||||
def raise_unsupported():
|
||||
raise io.UnsupportedOperation
|
||||
|
||||
|
|
@ -113,8 +147,7 @@ class BuildTriggerHandler(object):
|
|||
|
||||
def handle_trigger_request(self):
|
||||
"""
|
||||
Transform the incoming request data into a set of actions. Returns a tuple
|
||||
of usefiles resource id, docker tags, build name, and resource subdir.
|
||||
Transform the incoming request data into a set of actions. Returns a PreparedBuild.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -142,7 +175,7 @@ class BuildTriggerHandler(object):
|
|||
|
||||
def manual_start(self, run_parameters=None):
|
||||
"""
|
||||
Manually creates a repository build for this trigger.
|
||||
Manually creates a repository build for this trigger. Returns a PreparedBuild.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -166,7 +199,7 @@ class BuildTriggerHandler(object):
|
|||
if subc.service_name() == trigger.service.name:
|
||||
return subc(trigger, override_config)
|
||||
|
||||
raise InvalidServiceException('Unable to find service: %s' % service)
|
||||
raise InvalidServiceException('Unable to find service: %s' % trigger.service.name)
|
||||
|
||||
def put_config_key(self, key, value):
|
||||
""" Updates a config key in the trigger, saving it to the DB. """
|
||||
|
|
@ -272,7 +305,6 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
config['hook_id'] = data['id']
|
||||
return config, {'private_key': private_key}
|
||||
|
||||
|
||||
def deactivate(self):
|
||||
config = self.config
|
||||
repository = self._get_repository_client()
|
||||
|
|
@ -294,7 +326,6 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
return config
|
||||
|
||||
|
||||
def list_build_sources(self):
|
||||
bitbucket_client = self._get_authorized_client()
|
||||
(result, data, err_msg) = bitbucket_client.get_visible_repositories()
|
||||
|
|
@ -321,12 +352,16 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
return namespaces.values()
|
||||
|
||||
def list_build_subdirs(self):
|
||||
config = self.config
|
||||
repository = self._get_repository_client()
|
||||
(result, data, err_msg) = repository.get_path_contents('', revision='master')
|
||||
|
||||
# Find the first matching branch.
|
||||
repo_branches = self.list_field_values('branch_name') or []
|
||||
branches = find_matching_branches(config, repo_branches)
|
||||
(result, data, err_msg) = repository.get_path_contents('', revision=branches[0])
|
||||
if not result:
|
||||
raise RepositoryReadException(err_msg)
|
||||
|
||||
|
||||
files = set([f['path'] for f in data['files']])
|
||||
if 'Dockerfile' in files:
|
||||
return ['/']
|
||||
|
|
@ -392,29 +427,114 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
return None
|
||||
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
return
|
||||
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
def _prepare_build(self, commit_sha, ref, is_manual):
|
||||
config = self.config
|
||||
repository = self._get_repository_client()
|
||||
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
||||
# Lookup the branch to build.
|
||||
master_branch = 'master'
|
||||
# Lookup the default branch associated with the repository. We use this when building
|
||||
# the tags.
|
||||
default_branch = ''
|
||||
(result, data, _) = repository.get_main_branch()
|
||||
if result:
|
||||
master_branch = data['name']
|
||||
default_branch = data['name']
|
||||
|
||||
branch_name = run_parameters.get('branch_name') or master_branch
|
||||
# Lookup the commit sha.
|
||||
(result, data, _) = repository.changesets().get(commit_sha)
|
||||
if not result:
|
||||
raise TriggerStartException('Could not lookup commit SHA')
|
||||
|
||||
# Find the SHA for the branch.
|
||||
# TODO
|
||||
return None
|
||||
namespace = repository.namespace
|
||||
name = repository.repository_name
|
||||
|
||||
commit_info = {
|
||||
'url': 'https://bitbucket.org/%s/%s/commits/%s' % (namespace, name, commit_sha),
|
||||
'message': data['message'],
|
||||
'date': data['timestamp']
|
||||
}
|
||||
|
||||
# Try to lookup the author by email address. The raw_author field (if it exists) is returned
|
||||
# in the form: "Joseph Schorr <joseph.schorr@coreos.com>"
|
||||
if data.get('raw_author'):
|
||||
match = re.compile(r'.*<(.+)>').match(data['raw_author'])
|
||||
if match:
|
||||
email_address = match.group(1)
|
||||
bitbucket_client = self._get_authorized_client()
|
||||
(result, data, _) = bitbucket_client.accounts().get_profile(email_address)
|
||||
if result:
|
||||
commit_info['author'] = {
|
||||
'username': data['user']['username'],
|
||||
'url': 'https://bitbucket.org/%s/' % data['user']['username'],
|
||||
'avatar_url': data['user']['avatar']
|
||||
}
|
||||
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'ref': ref,
|
||||
'default_branch': default_branch,
|
||||
'git_url': 'git@bitbucket.org:%s/%s.git' % (namespace, name),
|
||||
'commit_info': commit_info
|
||||
}
|
||||
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
prepared.tags_from_ref(ref, default_branch)
|
||||
prepared.name_from_sha(commit_sha)
|
||||
prepared.subdirectory = config['subdir']
|
||||
prepared.metadata = metadata
|
||||
prepared.is_manual = is_manual
|
||||
|
||||
return prepared
|
||||
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
# Parse the JSON payload.
|
||||
payload_json = request.form.get('payload')
|
||||
if not payload_json:
|
||||
raise SkipRequestException()
|
||||
|
||||
try:
|
||||
payload = json.loads(payload_json)
|
||||
except ValueError:
|
||||
raise SkipRequestException()
|
||||
|
||||
logger.debug('BitBucket trigger payload %s', payload)
|
||||
|
||||
# Make sure we have a commit in the payload.
|
||||
if not payload.get('commits'):
|
||||
raise SkipRequestException()
|
||||
|
||||
# Check if this build should be skipped by commit message.
|
||||
commit = payload['commits'][0]
|
||||
commit_message = commit['message']
|
||||
if should_skip_commit(commit_message):
|
||||
raise SkipRequestException()
|
||||
|
||||
# Check to see if this build should be skipped by ref.
|
||||
ref = 'refs/heads/' + commit['branch'] if commit.get('branch') else 'refs/tags/' + commit['tag']
|
||||
raise_if_skipped(self.config, ref)
|
||||
|
||||
commit_sha = commit['node']
|
||||
return self._prepare_build(commit_sha, ref, False)
|
||||
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
run_parameters = run_parameters or {}
|
||||
repository = self._get_repository_client()
|
||||
|
||||
# Find the branch to build.
|
||||
branch_name = run_parameters.get('branch_name')
|
||||
(result, data, _) = repository.get_main_branch()
|
||||
if result:
|
||||
branch_name = data['name'] or branch_name
|
||||
|
||||
# Lookup the commit SHA for the branch.
|
||||
(result, data, _) = repository.get_branches()
|
||||
if not result or not branch_name in data:
|
||||
raise TriggerStartException('Could not find branch commit SHA')
|
||||
|
||||
commit_sha = data[branch_name]['node']
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
return self._prepare_build(commit_sha, ref, True)
|
||||
|
||||
|
||||
class GithubBuildTrigger(BuildTriggerHandler):
|
||||
|
|
@ -543,18 +663,6 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
return repos_by_org
|
||||
|
||||
@staticmethod
|
||||
def matches_ref(ref, regex):
|
||||
match_string = ref.split('/', 1)[1]
|
||||
if not regex:
|
||||
return False
|
||||
|
||||
m = regex.match(match_string)
|
||||
if not m:
|
||||
return False
|
||||
|
||||
return len(m.group(0)) == len(match_string)
|
||||
|
||||
def list_build_subdirs(self):
|
||||
config = self.config
|
||||
gh_client = self._get_client()
|
||||
|
|
@ -564,15 +672,8 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
repo = gh_client.get_repo(source)
|
||||
|
||||
# Find the first matching branch.
|
||||
branches = None
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
branches = [branch.name for branch in repo.get_branches()
|
||||
if GithubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
|
||||
except:
|
||||
pass
|
||||
|
||||
repo_branches = self.list_field_values('branch_name') or []
|
||||
branches = find_matching_branches(config, repo_branches)
|
||||
branches = branches or [repo.default_branch or 'master']
|
||||
default_commit = repo.get_branch(branches[0]).commit
|
||||
commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
|
||||
|
|
@ -691,37 +792,31 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
return tarball_subdir, dockerfile_id
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _prepare_build(trigger, config, repo, commit_sha, build_name, ref, git_url):
|
||||
repo_subdir = config['subdir']
|
||||
joined_subdir = repo_subdir
|
||||
dockerfile_id = None
|
||||
def _prepare_build(self, repo, ref, commit_sha, is_manual):
|
||||
config = self.config
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
|
||||
if trigger.private_key is None:
|
||||
# If the trigger isn't using git, prepare the buildpack.
|
||||
# If the trigger isn't using git, prepare the buildpack.
|
||||
if self.trigger.private_key is None:
|
||||
tarball_subdir, dockerfile_id = GithubBuildTrigger._prepare_tarball(repo, commit_sha)
|
||||
logger.debug('Successfully prepared job')
|
||||
|
||||
# Join provided subdir with the tarball subdir.
|
||||
joined_subdir = os.path.join(tarball_subdir, repo_subdir)
|
||||
prepared.subdirectory = os.path.join(tarball_subdir, config['subdir'])
|
||||
prepared.dockerfile_id = dockerfile_id
|
||||
else:
|
||||
prepared.subdirectory = config['subdir']
|
||||
|
||||
logger.debug('Final subdir: %s', joined_subdir)
|
||||
# Set the name.
|
||||
prepared.name_from_sha(commit_sha)
|
||||
|
||||
# compute the tag(s)
|
||||
branch = ref.split('/')[-1]
|
||||
tags = {branch}
|
||||
# Set the tag(s).
|
||||
prepared.tags_from_ref(ref, repo.default_branch)
|
||||
|
||||
if branch == repo.default_branch:
|
||||
tags.add('latest')
|
||||
|
||||
logger.debug('Pushing to tags: %s', tags)
|
||||
|
||||
# compute the metadata
|
||||
# Build and set the metadata.
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'ref': ref,
|
||||
'default_branch': repo.default_branch,
|
||||
'git_url': git_url,
|
||||
'git_url': repo.git_url,
|
||||
}
|
||||
|
||||
# add the commit info.
|
||||
|
|
@ -729,71 +824,63 @@ class GithubBuildTrigger(BuildTriggerHandler):
|
|||
if commit_info is not None:
|
||||
metadata['commit_info'] = commit_info
|
||||
|
||||
return dockerfile_id, list(tags), build_name, joined_subdir, metadata
|
||||
prepared.metadata = metadata
|
||||
prepared.is_manual = is_manual
|
||||
return prepared
|
||||
|
||||
@staticmethod
|
||||
def get_display_name(sha):
|
||||
return sha[0:7]
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
# Check the payload to see if we should skip it based on the lack of a head_commit.
|
||||
payload = request.get_json()
|
||||
if not payload or payload.get('head_commit') is None:
|
||||
raise SkipRequestException()
|
||||
|
||||
# This is for GitHub's probing/testing.
|
||||
if 'zen' in payload:
|
||||
raise ValidationRequestException()
|
||||
|
||||
logger.debug('Payload %s', payload)
|
||||
logger.debug('GitHub trigger payload %s', payload)
|
||||
|
||||
ref = payload['ref']
|
||||
commit_sha = payload['head_commit']['id']
|
||||
commit_message = payload['head_commit'].get('message', '')
|
||||
git_url = payload['repository']['git_url']
|
||||
|
||||
config = self.config
|
||||
if 'branchtag_regex' in config:
|
||||
try:
|
||||
regex = re.compile(config['branchtag_regex'])
|
||||
except:
|
||||
regex = re.compile('.*')
|
||||
|
||||
if not GithubBuildTrigger.matches_ref(ref, regex):
|
||||
raise SkipRequestException()
|
||||
|
||||
# Check if this build should be skipped by commit message.
|
||||
if should_skip_commit(commit_message):
|
||||
raise SkipRequestException()
|
||||
|
||||
short_sha = GithubBuildTrigger.get_display_name(commit_sha)
|
||||
# Check to see if this build should be skipped by ref.
|
||||
raise_if_skipped(self.config, ref)
|
||||
|
||||
gh_client = self._get_client()
|
||||
|
||||
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||
payload['repository']['name'])
|
||||
repo = gh_client.get_repo(repo_full_name)
|
||||
|
||||
logger.debug('Github repo: %s', repo)
|
||||
|
||||
return GithubBuildTrigger._prepare_build(self.trigger, config, repo, commit_sha, short_sha,
|
||||
ref, git_url)
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
config = self.config
|
||||
try:
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
|
||||
payload['repository']['name'])
|
||||
|
||||
gh_client = self._get_client()
|
||||
repo = gh_client.get_repo(source)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
branch_sha = branch.commit.sha
|
||||
short_sha = GithubBuildTrigger.get_display_name(branch_sha)
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
git_url = repo.git_url
|
||||
repo = gh_client.get_repo(repo_full_name)
|
||||
|
||||
return self._prepare_build(self.trigger, config, repo, branch_sha, short_sha, ref, git_url)
|
||||
return self._prepare_build(repo, ref, commit_sha, False)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
config = self.config
|
||||
source = config['build_source']
|
||||
run_parameters = run_parameters or {}
|
||||
|
||||
try:
|
||||
gh_client = self._get_client()
|
||||
|
||||
# Lookup the branch and its associated current SHA.
|
||||
repo = gh_client.get_repo(source)
|
||||
branch_name = run_parameters.get('branch_name') or repo.default_branch
|
||||
branch = repo.get_branch(branch_name)
|
||||
commit_sha = branch.commit.sha
|
||||
ref = 'refs/heads/%s' % (branch_name)
|
||||
|
||||
return self._prepare_build(repo, ref, commit_sha, True)
|
||||
except GithubException as ghe:
|
||||
raise TriggerStartException(ghe.data['message'])
|
||||
|
||||
def list_field_values(self, field_name):
|
||||
if field_name == 'refs':
|
||||
|
|
@ -922,24 +1009,50 @@ class CustomBuildTrigger(BuildTriggerHandler):
|
|||
return metadata
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
# Skip if there is no payload.
|
||||
payload = request.get_json()
|
||||
if not payload:
|
||||
raise SkipRequestException()
|
||||
|
||||
logger.debug('Payload %s', payload)
|
||||
|
||||
# Skip if the commit message matches.
|
||||
metadata = self._metadata_from_payload(payload)
|
||||
if should_skip_commit(metadata.get('commit_info', {}).get('message', '')):
|
||||
raise SkipRequestException()
|
||||
|
||||
# The build source is the canonical git URL used to clone.
|
||||
config = self.config
|
||||
metadata['git_url'] = config['build_source']
|
||||
|
||||
branch = metadata['ref'].split('/')[-1]
|
||||
tags = {branch}
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
prepared.tags_from_ref(metadata['ref'])
|
||||
prepared.name_from_sha(metadata['commit_sha'])
|
||||
prepared.subdirectory = config['subdir']
|
||||
prepared.metadata = metadata
|
||||
|
||||
build_name = metadata['commit_sha'][:6]
|
||||
dockerfile_id = None
|
||||
return prepared
|
||||
|
||||
return dockerfile_id, tags, build_name, config['subdir'], metadata
|
||||
def manual_start(self, run_parameters=None):
|
||||
# commit_sha is the only required parameter
|
||||
commit_sha = run_parameters.get('commit_sha')
|
||||
if commit_sha is None:
|
||||
raise TriggerStartException('missing required parameter')
|
||||
|
||||
config = self.config
|
||||
metadata = {
|
||||
'commit_sha': commit_sha,
|
||||
'git_url': config['build_source'],
|
||||
}
|
||||
|
||||
prepared = PreparedBuild(self.trigger)
|
||||
prepared.tags = [commit_sha]
|
||||
prepared.name_from_sha(commit_sha)
|
||||
prepared.subdirectory = config['subdir']
|
||||
prepared.metadata = metadata
|
||||
prepared.is_manual = True
|
||||
|
||||
return prepared
|
||||
|
||||
def activate(self, standard_webhook_url):
|
||||
config = self.config
|
||||
|
|
@ -962,19 +1075,3 @@ class CustomBuildTrigger(BuildTriggerHandler):
|
|||
config.pop('credentials', None)
|
||||
self.config = config
|
||||
return config
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
# commit_sha is the only required parameter
|
||||
if 'commit_sha' not in run_parameters:
|
||||
raise TriggerStartException('missing required parameter')
|
||||
|
||||
config = self.config
|
||||
dockerfile_id = None
|
||||
tags = {run_parameters['commit_sha']}
|
||||
build_name = run_parameters['commit_sha']
|
||||
metadata = {
|
||||
'commit_sha': run_parameters['commit_sha'],
|
||||
'git_url': config['build_source'],
|
||||
}
|
||||
|
||||
return dockerfile_id, list(tags), build_name, config['subdir'], metadata
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from util.useremails import send_invoice_email, send_subscription_change, send_p
|
|||
from util.http import abort
|
||||
from endpoints.trigger import (BuildTriggerHandler, ValidationRequestException,
|
||||
SkipRequestException, InvalidPayloadException)
|
||||
from endpoints.common import start_build
|
||||
from endpoints.building import start_build
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -87,8 +87,7 @@ def build_trigger_webhook(trigger_uuid, **kwargs):
|
|||
|
||||
logger.debug('Passing webhook request to handler %s', handler)
|
||||
try:
|
||||
specs = handler.handle_trigger_request(request)
|
||||
dockerfile_id, tags, name, subdir, metadata = specs
|
||||
prepared = handler.handle_trigger_request(request)
|
||||
except ValidationRequestException:
|
||||
# This was just a validation request, we don't need to build anything
|
||||
return make_response('Okay')
|
||||
|
|
@ -101,8 +100,7 @@ def build_trigger_webhook(trigger_uuid, **kwargs):
|
|||
|
||||
pull_robot_name = model.get_pull_robot_name(trigger)
|
||||
repo = model.get_repository(namespace, repository)
|
||||
start_build(repo, dockerfile_id, tags, name, subdir, False, trigger,
|
||||
pull_robot_name=pull_robot_name, trigger_metadata=metadata)
|
||||
start_build(repo, prepared, pull_robot_name=pull_robot_name)
|
||||
|
||||
return make_response('Okay')
|
||||
|
||||
|
|
|
|||
Reference in a new issue