This repository has been archived on 2020-03-24. You can view files and clone it, but cannot push or open issues or pull requests.
quay/buildtrigger/gitlabhandler.py
Jimmy Zelinskie bcea268fcb use app.gitlab_trigger for config data
This includes defaults and makes the structure of the Gitlab trigger
parallel the GitHub trigger.
2016-03-18 14:56:09 -04:00

432 lines
13 KiB
Python

import logging
from functools import wraps
from urlparse import urljoin
from app import app, gitlab_trigger
from jsonschema import validate
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
TriggerDeactivationException, TriggerStartException,
SkipRequestException, InvalidPayloadException,
determine_build_ref, raise_if_skipped_build,
find_matching_branches)
from buildtrigger.basehandler import BuildTriggerHandler
from util.security.ssh import generate_ssh_keypair
from util.dict_wrappers import JSONPathDict, SafeDictSetter
from endpoints.api import ExternalServiceTimeout
import gitlab
import requests
logger = logging.getLogger(__name__)
GITLAB_WEBHOOK_PAYLOAD_SCHEMA = {
'type': 'object',
'properties': {
'ref': {
'type': 'string',
},
'checkout_sha': {
'type': 'string',
},
'repository': {
'type': 'object',
'properties': {
'git_ssh_url': {
'type': 'string',
},
},
'required': ['git_ssh_url'],
},
'commits': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'url': {
'type': 'string',
},
'message': {
'type': 'string',
},
'timestamp': {
'type': 'string',
},
'author': {
'type': 'object',
'properties': {
'email': {
'type': 'string',
},
},
'required': ['email'],
},
},
'required': ['url', 'message', 'timestamp'],
},
'minItems': 1,
}
},
'required': ['ref', 'checkout_sha', 'repository'],
}
def _catch_timeouts(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except requests.exceptions.Timeout:
msg = 'Request to the GitLab API timed out'
logger.exception(msg)
raise ExternalServiceTimeout(msg)
return wrapper
def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None):
""" Returns the Gitlab webhook JSON payload transformed into our own payload
format. If the gl_payload is not valid, returns None.
"""
try:
validate(gl_payload, GITLAB_WEBHOOK_PAYLOAD_SCHEMA)
except Exception as exc:
raise InvalidPayloadException(exc.message)
payload = JSONPathDict(gl_payload)
config = SafeDictSetter()
config['commit'] = payload['checkout_sha']
config['ref'] = payload['ref']
config['default_branch'] = default_branch
config['git_url'] = payload['repository.git_ssh_url']
config['commit_info.url'] = payload['commits[0].url']
config['commit_info.message'] = payload['commits[0].message']
config['commit_info.date'] = payload['commits[0].timestamp']
# Note: Gitlab does not send full user information with the payload, so we have to
# (optionally) look it up.
author_email = payload['commits[0].author.email']
if lookup_user and author_email:
author_info = lookup_user(author_email)
if author_info:
config['commit_info.author.username'] = author_info['username']
config['commit_info.author.url'] = author_info['html_url']
config['commit_info.author.avatar_url'] = author_info['avatar_url']
return config.dict_value()
class GitLabBuildTrigger(BuildTriggerHandler):
"""
BuildTrigger for GitLab.
"""
@classmethod
def service_name(cls):
return 'gitlab'
def _get_authorized_client(self):
auth_token = self.auth_token or 'invalid'
return gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=5)
def is_active(self):
return 'hook_id' in self.config
@_catch_timeouts
def activate(self, standard_webhook_url):
config = self.config
new_build_source = config['build_source']
gl_client = self._get_authorized_client()
# Find the GitLab repository.
repository = gl_client.getproject(new_build_source)
if repository is False:
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
raise TriggerActivationException(msg)
# Add a deploy key to the repository.
public_key, private_key = generate_ssh_keypair()
config['credentials'] = [
{
'name': 'SSH Public Key',
'value': public_key,
},
]
key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'],
public_key)
if key is False:
msg = 'Unable to add deploy key to repository: %s' % new_build_source
raise TriggerActivationException(msg)
config['key_id'] = key['id']
# Add the webhook to the GitLab repository.
hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True)
if hook is False:
msg = 'Unable to create webhook on repository: %s' % new_build_source
raise TriggerActivationException(msg)
config['hook_id'] = hook['id']
self.config = config
return config, {'private_key': private_key}
def deactivate(self):
config = self.config
gl_client = self._get_authorized_client()
# Find the GitLab repository.
repository = gl_client.getproject(config['build_source'])
if repository is False:
msg = 'Unable to find GitLab repository for source: %s' % config['build_source']
raise TriggerDeactivationException(msg)
# Remove the webhook.
success = gl_client.deleteprojecthook(repository['id'], config['hook_id'])
if success is False:
msg = 'Unable to remove hook: %s' % config['hook_id']
raise TriggerDeactivationException(msg)
config.pop('hook_id', None)
# Remove the key
success = gl_client.deletedeploykey(repository['id'], config['key_id'])
if success is False:
msg = 'Unable to remove deploy key: %s' % config['key_id']
raise TriggerDeactivationException(msg)
config.pop('key_id', None)
self.config = config
return config
@_catch_timeouts
def list_build_sources(self):
gl_client = self._get_authorized_client()
current_user = gl_client.currentuser()
if current_user is False:
raise RepositoryReadException('Unable to get current user')
repositories = gl_client.getprojects()
if repositories is False:
raise RepositoryReadException('Unable to list user repositories')
namespaces = {}
for repo in repositories:
owner = repo['namespace']['name']
if not owner in namespaces:
namespaces[owner] = {
'personal': owner == current_user['username'],
'repos': [],
'info': {
'name': owner,
}
}
namespaces[owner]['repos'].append(repo['path_with_namespace'])
return namespaces.values()
@_catch_timeouts
def list_build_subdirs(self):
config = self.config
gl_client = self._get_authorized_client()
new_build_source = config['build_source']
repository = gl_client.getproject(new_build_source)
if repository is False:
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
raise RepositoryReadException(msg)
repo_branches = gl_client.getbranches(repository['id'])
if repo_branches is False:
msg = 'Unable to find GitLab branches for source: %s' % new_build_source
raise RepositoryReadException(msg)
branches = [branch['name'] for branch in repo_branches]
branches = find_matching_branches(config, branches)
branches = branches or [repository['default_branch'] or 'master']
repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0])
if repo_tree is False:
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
raise RepositoryReadException(msg)
for node in repo_tree:
if node['name'] == 'Dockerfile':
return ['/']
return []
@_catch_timeouts
def load_dockerfile_contents(self):
gl_client = self._get_authorized_client()
path = self.get_dockerfile_path()
repository = gl_client.getproject(self.config['build_source'])
if repository is False:
return None
branches = self.list_field_values('branch_name')
branches = find_matching_branches(self.config, branches)
if branches == []:
return None
branch_name = branches[0]
if repository['default_branch'] in branches:
branch_name = repository['default_branch']
contents = gl_client.getrawfile(repository['id'], branch_name, path)
if contents is False:
return None
return contents
@_catch_timeouts
def list_field_values(self, field_name, limit=None):
if field_name == 'refs':
branches = self.list_field_values('branch_name')
tags = self.list_field_values('tag_name')
return ([{'kind': 'branch', 'name': b} for b in branches] +
[{'kind': 'tag', 'name': t} for t in tags])
gl_client = self._get_authorized_client()
repo = gl_client.getproject(self.config['build_source'])
if repo is False:
return []
if field_name == 'tag_name':
tags = gl_client.getrepositorytags(repo['id'])
if tags is False:
return []
if limit:
tags = tags[0:limit]
return [tag['name'] for tag in tags]
if field_name == 'branch_name':
branches = gl_client.getbranches(repo['id'])
if branches is False:
return []
if limit:
branches = branches[0:limit]
return [branch['name'] for branch in branches]
return None
def get_repository_url(self):
return gitlab_trigger.get_public_url(self.config['build_source'])
@_catch_timeouts
def lookup_user(self, email):
gl_client = self._get_authorized_client()
try:
[user] = gl_client.getusers(search=email)
return {
'username': user['username'],
'html_url': gl_client.host + '/' + user['username'],
'avatar_url': user['avatar_url']
}
except ValueError:
return None
@_catch_timeouts
def get_metadata_for_commit(self, commit_sha, ref, repo):
gl_client = self._get_authorized_client()
commit = gl_client.getrepositorycommit(repo['id'], commit_sha)
metadata = {
'commit': commit['id'],
'ref': ref,
'default_branch': repo['default_branch'],
'git_url': repo['ssh_url_to_repo'],
'commit_info': {
'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'],
'message': commit['message'],
'date': commit['committed_date'],
},
}
committer = None
if 'committer_email' in commit:
committer = self.lookup_user(commit['committer_email'])
author = None
if 'author_email' in commit:
author = self.lookup_user(commit['author_email'])
if committer is not None:
metadata['commit_info']['committer'] = {
'username': committer['username'],
'avatar_url': committer['avatar_url'],
'url': gl_client.host + '/' + committer['username'],
}
if author is not None:
metadata['commit_info']['author'] = {
'username': author['username'],
'avatar_url': author['avatar_url'],
'url': gl_client.host + '/' + author['username']
}
return metadata
@_catch_timeouts
def manual_start(self, run_parameters=None):
gl_client = self._get_authorized_client()
repo = gl_client.getproject(self.config['build_source'])
if repo is False:
raise TriggerStartException('Could not find repository')
def get_tag_sha(tag_name):
tags = gl_client.getrepositorytags(repo['id'])
if tags is False:
raise TriggerStartException('Could not find tags')
for tag in tags:
if tag['name'] == tag_name:
return tag['commit']['id']
raise TriggerStartException('Could not find commit')
def get_branch_sha(branch_name):
branch = gl_client.getbranch(repo['id'], branch_name)
if branch is False:
raise TriggerStartException('Could not find branch')
return branch['commit']['id']
# Find the branch or tag to build.
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
repo['default_branch'])
metadata = self.get_metadata_for_commit(commit_sha, ref, repo)
return self.prepare_build(metadata, is_manual=True)
@_catch_timeouts
def handle_trigger_request(self, request):
payload = request.get_json()
if not payload:
raise SkipRequestException()
# Lookup the default branch.
default_branch = None
gl_client = self._get_authorized_client()
repo = gl_client.getproject(self.config['build_source'])
if repo is not False:
default_branch = repo['default_branch']
lookup_user = self.lookup_user
logger.debug('GitLab trigger payload %s', payload)
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
lookup_user=lookup_user)
prepared = self.prepare_build(metadata)
# Check if we should skip this build.
raise_if_skipped_build(prepared, self.config)
return prepared