Start refactoring of the trigger system:
- Move each trigger handler into its own file - Add dictionary helper classes for easier reading and writing of dict-based data - Extract the web hook payload -> internal representation building for each trigger system - Add tests for this transformation - Remove support for Github archived-based building
This commit is contained in:
Joseph Schorr
parent
2ff77df946
commit
49b575afb6
25 changed files with 2449 additions and 1602 deletions
359
buildtrigger/gitlabhandler.py
Normal file
359
buildtrigger/gitlabhandler.py
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
import logging
|
||||
|
||||
from app import app
|
||||
|
||||
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
||||
TriggerDeactivationException, TriggerStartException,
|
||||
EmptyRepositoryException, ValidationRequestException,
|
||||
SkipRequestException,
|
||||
determine_build_ref, raise_if_skipped_build,
|
||||
find_matching_branches)
|
||||
|
||||
from buildtrigger.basehandler import BuildTriggerHandler
|
||||
|
||||
from util.security.ssh import generate_ssh_keypair
|
||||
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
||||
|
||||
import gitlab
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None):
|
||||
""" Returns the Gitlab webhook JSON payload transformed into our own payload
|
||||
format. If the gl_payload is not valid, returns None.
|
||||
"""
|
||||
# TODO(jschorr): Validate payload JSON
|
||||
payload = JSONPathDict(gl_payload)
|
||||
|
||||
config = SafeDictSetter()
|
||||
config['commit'] = payload['checkout_sha']
|
||||
config['ref'] = payload['ref']
|
||||
config['default_branch'] = default_branch
|
||||
config['git_url'] = payload['repository.git_ssh_url']
|
||||
|
||||
config['commit_info.url'] = payload['commits[0].url']
|
||||
config['commit_info.message'] = payload['commits[0].message']
|
||||
config['commit_info.date'] = payload['commits[0].timestamp']
|
||||
|
||||
# Note: Gitlab does not send full user information with the payload, so we have to
|
||||
# (optionally) look it up.
|
||||
author_email = payload['commits[0].author.email']
|
||||
if lookup_user and author_email:
|
||||
author_info = lookup_user(author_email)
|
||||
if author_info:
|
||||
config['commit_info.author.username'] = author_info['username']
|
||||
config['commit_info.author.url'] = author_info['html_url']
|
||||
config['commit_info.author.avatar_url'] = author_info['avatar_url']
|
||||
|
||||
return config.dict_value()
|
||||
|
||||
|
||||
class GitLabBuildTrigger(BuildTriggerHandler):
|
||||
"""
|
||||
BuildTrigger for GitLab.
|
||||
"""
|
||||
@classmethod
|
||||
def service_name(cls):
|
||||
return 'gitlab'
|
||||
|
||||
def _get_authorized_client(self):
|
||||
host = app.config.get('GITLAB_TRIGGER_CONFIG', {}).get('GITLAB_ENDPOINT', '')
|
||||
auth_token = self.auth_token or 'invalid'
|
||||
return gitlab.Gitlab(host, oauth_token=auth_token)
|
||||
|
||||
def is_active(self):
|
||||
return 'hook_id' in self.config
|
||||
|
||||
def activate(self, standard_webhook_url):
|
||||
config = self.config
|
||||
new_build_source = config['build_source']
|
||||
gl_client = self._get_authorized_client()
|
||||
|
||||
# Find the GitLab repository.
|
||||
repository = gl_client.getproject(new_build_source)
|
||||
if repository is False:
|
||||
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
# Add a deploy key to the repository.
|
||||
public_key, private_key = generate_ssh_keypair()
|
||||
config['credentials'] = [
|
||||
{
|
||||
'name': 'SSH Public Key',
|
||||
'value': public_key,
|
||||
},
|
||||
]
|
||||
key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'],
|
||||
public_key)
|
||||
if key is False:
|
||||
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
config['key_id'] = key['id']
|
||||
|
||||
# Add the webhook to the GitLab repository.
|
||||
hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True)
|
||||
if hook is False:
|
||||
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
config['hook_id'] = hook['id']
|
||||
self.config = config
|
||||
return config, {'private_key': private_key}
|
||||
|
||||
def deactivate(self):
|
||||
config = self.config
|
||||
gl_client = self._get_authorized_client()
|
||||
|
||||
# Find the GitLab repository.
|
||||
repository = gl_client.getproject(config['build_source'])
|
||||
if repository is False:
|
||||
msg = 'Unable to find GitLab repository for source: %s' % config['build_source']
|
||||
raise TriggerDeactivationException(msg)
|
||||
|
||||
# Remove the webhook.
|
||||
success = gl_client.deleteprojecthook(repository['id'], config['hook_id'])
|
||||
if success is False:
|
||||
msg = 'Unable to remove hook: %s' % config['hook_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
config.pop('hook_id', None)
|
||||
|
||||
# Remove the key
|
||||
success = gl_client.deletedeploykey(repository['id'], config['key_id'])
|
||||
if success is False:
|
||||
msg = 'Unable to remove deploy key: %s' % config['key_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
config.pop('key_id', None)
|
||||
|
||||
self.config = config
|
||||
|
||||
return config
|
||||
|
||||
def list_build_sources(self):
|
||||
gl_client = self._get_authorized_client()
|
||||
current_user = gl_client.currentuser()
|
||||
if current_user is False:
|
||||
raise RepositoryReadException('Unable to get current user')
|
||||
|
||||
repositories = gl_client.getprojects()
|
||||
if repositories is False:
|
||||
raise RepositoryReadException('Unable to list user repositories')
|
||||
|
||||
namespaces = {}
|
||||
for repo in repositories:
|
||||
owner = repo['namespace']['name']
|
||||
if not owner in namespaces:
|
||||
namespaces[owner] = {
|
||||
'personal': owner == current_user['username'],
|
||||
'repos': [],
|
||||
'info': {
|
||||
'name': owner,
|
||||
}
|
||||
}
|
||||
|
||||
namespaces[owner]['repos'].append(repo['path_with_namespace'])
|
||||
|
||||
return namespaces.values()
|
||||
|
||||
def list_build_subdirs(self):
|
||||
config = self.config
|
||||
gl_client = self._get_authorized_client()
|
||||
new_build_source = config['build_source']
|
||||
|
||||
repository = gl_client.getproject(new_build_source)
|
||||
if repository is False:
|
||||
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
||||
raise RepositoryReadException(msg)
|
||||
|
||||
repo_branches = gl_client.getbranches(repository['id'])
|
||||
if repo_branches is False:
|
||||
msg = 'Unable to find GitLab branches for source: %s' % new_build_source
|
||||
raise RepositoryReadException(msg)
|
||||
|
||||
branches = [branch['name'] for branch in repo_branches]
|
||||
branches = find_matching_branches(config, branches)
|
||||
branches = branches or [repository['default_branch'] or 'master']
|
||||
|
||||
repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0])
|
||||
if repo_tree is False:
|
||||
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
|
||||
raise RepositoryReadException(msg)
|
||||
|
||||
for node in repo_tree:
|
||||
if node['name'] == 'Dockerfile':
|
||||
return ['/']
|
||||
|
||||
return []
|
||||
|
||||
def load_dockerfile_contents(self):
|
||||
gl_client = self._get_authorized_client()
|
||||
path = self.get_dockerfile_path()
|
||||
|
||||
repository = gl_client.getproject(self.config['build_source'])
|
||||
if repository is False:
|
||||
return None
|
||||
|
||||
branches = self.list_field_values('branch_name')
|
||||
branches = find_matching_branches(self.config, branches)
|
||||
if branches == []:
|
||||
return None
|
||||
|
||||
branch_name = branches[0]
|
||||
if repository['default_branch'] in branches:
|
||||
branch_name = repository['default_branch']
|
||||
|
||||
contents = gl_client.getrawfile(repository['id'], branch_name, path)
|
||||
if contents is False:
|
||||
return None
|
||||
|
||||
return contents
|
||||
|
||||
def list_field_values(self, field_name, limit=None):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values('branch_name')
|
||||
tags = self.list_field_values('tag_name')
|
||||
|
||||
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
||||
[{'kind': 'tag', 'name': t} for t in tags])
|
||||
|
||||
gl_client = self._get_authorized_client()
|
||||
repo = gl_client.getproject(self.config['build_source'])
|
||||
if repo is False:
|
||||
return []
|
||||
|
||||
if field_name == 'tag_name':
|
||||
tags = gl_client.getrepositorytags(repo['id'])
|
||||
if tags is False:
|
||||
return []
|
||||
|
||||
if limit:
|
||||
tags = tags[0:limit]
|
||||
|
||||
return [tag['name'] for tag in tags]
|
||||
|
||||
if field_name == 'branch_name':
|
||||
branches = gl_client.getbranches(repo['id'])
|
||||
if branches is False:
|
||||
return []
|
||||
|
||||
if limit:
|
||||
branches = branches[0:limit]
|
||||
|
||||
return [branch['name'] for branch in branches]
|
||||
|
||||
return None
|
||||
|
||||
def get_repository_url(self):
|
||||
gl_client = self._get_authorized_client()
|
||||
repository = gl_client.getproject(self.config['build_source'])
|
||||
if repository is False:
|
||||
return None
|
||||
|
||||
return '%s/%s' % (gl_client.host, repository['path_with_namespace'])
|
||||
|
||||
def lookup_user(self, email):
|
||||
gl_client = self._get_authorized_client()
|
||||
try:
|
||||
[user] = gl_client.getusers(search=email)
|
||||
|
||||
return {
|
||||
'username': user['username'],
|
||||
'html_url': gl_client.host + '/' + user['username'],
|
||||
'avatar_url': user['avatar_url']
|
||||
}
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def get_metadata_for_commit(self, commit_sha, ref, repo):
|
||||
gl_client = self._get_authorized_client()
|
||||
commit = gl_client.getrepositorycommit(repo['id'], commit_sha)
|
||||
|
||||
metadata = {
|
||||
'commit': commit['id'],
|
||||
'ref': ref,
|
||||
'default_branch': repo['default_branch'],
|
||||
'git_url': repo['ssh_url_to_repo'],
|
||||
'commit_info': {
|
||||
'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'],
|
||||
'message': commit['message'],
|
||||
'date': commit['committed_date'],
|
||||
},
|
||||
}
|
||||
|
||||
committer = None
|
||||
if 'committer_email' in commit:
|
||||
committer = self.lookup_user(commit['committer_email'])
|
||||
|
||||
author = None
|
||||
if 'author_email' in commit:
|
||||
author = self.lookup_user(commit['author_email'])
|
||||
|
||||
if committer is not None:
|
||||
metadata['commit_info']['committer'] = {
|
||||
'username': committer['username'],
|
||||
'avatar_url': committer['avatar_url'],
|
||||
'url': gl_client.host + '/' + committer['username'],
|
||||
}
|
||||
|
||||
if author is not None:
|
||||
metadata['commit_info']['author'] = {
|
||||
'username': author['username'],
|
||||
'avatar_url': author['avatar_url'],
|
||||
'url': gl_client.host + '/' + author['username']
|
||||
}
|
||||
|
||||
return metadata
|
||||
|
||||
def manual_start(self, run_parameters=None):
|
||||
gl_client = self._get_authorized_client()
|
||||
|
||||
repo = gl_client.getproject(self.config['build_source'])
|
||||
if repo is False:
|
||||
raise TriggerStartException('Could not find repository')
|
||||
|
||||
def get_tag_sha(tag_name):
|
||||
tags = gl_client.getrepositorytags(repo['id'])
|
||||
if tags is False:
|
||||
raise TriggerStartException('Could not find tags')
|
||||
|
||||
for tag in tags:
|
||||
if tag['name'] == tag_name:
|
||||
return tag['commit']['id']
|
||||
|
||||
raise TriggerStartException('Could not find commit')
|
||||
|
||||
def get_branch_sha(branch_name):
|
||||
branch = gl_client.getbranch(repo['id'], branch_name)
|
||||
if branch is False:
|
||||
raise TriggerStartException('Could not find branch')
|
||||
|
||||
return branch['commit']['id']
|
||||
|
||||
# Find the branch or tag to build.
|
||||
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
|
||||
repo['default_branch'])
|
||||
|
||||
metadata = self.get_metadata_for_commit(commit_sha, ref, repo)
|
||||
return self.prepare_build(metadata, is_manual=True)
|
||||
|
||||
def handle_trigger_request(self, request):
|
||||
payload = request.get_json()
|
||||
if not payload:
|
||||
raise SkipRequestException()
|
||||
|
||||
# Lookup the default branch.
|
||||
default_branch = None
|
||||
gl_client = self._get_authorized_client()
|
||||
repo = gl_client.getproject(self.config['build_source'])
|
||||
if repo is not False:
|
||||
default_branch = repo['default_branch']
|
||||
lookup_user = self.lookup_user
|
||||
|
||||
logger.debug('GitLab trigger payload %s', payload)
|
||||
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
|
||||
lookup_user=lookup_user)
|
||||
prepared = self.prepare_build(metadata)
|
||||
|
||||
# Check if we should skip this build.
|
||||
raise_if_skipped_build(prepared)
|
||||
return prepared
|
||||
Reference in a new issue