bf41aedc9c
This allows the repositories to be selected in the UI, if we are unsure whether the user has permission. Since gitlab will do the check anyway, this is safe, although not a great user experience if they chose an invalid repository, but we can't really do much about that.
568 lines
18 KiB
Python
568 lines
18 KiB
Python
import logging
|
|
import os
|
|
|
|
from calendar import timegm
|
|
from functools import wraps
|
|
|
|
import dateutil.parser
|
|
|
|
from app import app, gitlab_trigger
|
|
|
|
from jsonschema import validate
|
|
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
|
TriggerDeactivationException, TriggerStartException,
|
|
SkipRequestException, InvalidPayloadException,
|
|
determine_build_ref, raise_if_skipped_build,
|
|
find_matching_branches)
|
|
|
|
from buildtrigger.basehandler import BuildTriggerHandler
|
|
|
|
from util.security.ssh import generate_ssh_keypair
|
|
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
|
from endpoints.exception import ExternalServiceError
|
|
|
|
import gitlab
|
|
import requests
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
GITLAB_WEBHOOK_PAYLOAD_SCHEMA = {
|
|
'type': 'object',
|
|
'properties': {
|
|
'ref': {
|
|
'type': 'string',
|
|
},
|
|
'checkout_sha': {
|
|
'type': ['string', 'null'],
|
|
},
|
|
'repository': {
|
|
'type': 'object',
|
|
'properties': {
|
|
'git_ssh_url': {
|
|
'type': 'string',
|
|
},
|
|
},
|
|
'required': ['git_ssh_url'],
|
|
},
|
|
'commits': {
|
|
'type': 'array',
|
|
'items': {
|
|
'type': 'object',
|
|
'properties': {
|
|
'id': {
|
|
'type': 'string',
|
|
},
|
|
'url': {
|
|
'type': 'string',
|
|
},
|
|
'message': {
|
|
'type': 'string',
|
|
},
|
|
'timestamp': {
|
|
'type': 'string',
|
|
},
|
|
'author': {
|
|
'type': 'object',
|
|
'properties': {
|
|
'email': {
|
|
'type': 'string',
|
|
},
|
|
},
|
|
'required': ['email'],
|
|
},
|
|
},
|
|
'required': ['id', 'url', 'message', 'timestamp'],
|
|
},
|
|
},
|
|
},
|
|
'required': ['ref', 'checkout_sha', 'repository'],
|
|
}
|
|
|
|
_ACCESS_LEVEL_MAP = {
|
|
50: ("owner", True),
|
|
40: ("master", True),
|
|
30: ("developer", False),
|
|
20: ("reporter", False),
|
|
10: ("guest", False),
|
|
}
|
|
|
|
_PER_PAGE_COUNT = 20
|
|
|
|
|
|
def _catch_timeouts(func):
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except requests.exceptions.Timeout:
|
|
msg = 'Request to the GitLab API timed out'
|
|
logger.exception(msg)
|
|
raise ExternalServiceError(msg)
|
|
return wrapper
|
|
|
|
|
|
def _paginated_iterator(func, exc):
|
|
""" Returns an iterator over invocations of the given function, automatically handling
|
|
pagination.
|
|
"""
|
|
page = 0
|
|
while True:
|
|
result = func(page=page, per_page=_PER_PAGE_COUNT)
|
|
if result is False:
|
|
raise exc
|
|
|
|
counter = 0
|
|
for item in result:
|
|
yield item
|
|
counter = counter + 1
|
|
|
|
if counter < _PER_PAGE_COUNT:
|
|
break
|
|
|
|
page = page + 1
|
|
|
|
|
|
def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None,
|
|
lookup_commit=None):
|
|
""" Returns the Gitlab webhook JSON payload transformed into our own payload
|
|
format. If the gl_payload is not valid, returns None.
|
|
"""
|
|
try:
|
|
validate(gl_payload, GITLAB_WEBHOOK_PAYLOAD_SCHEMA)
|
|
except Exception as exc:
|
|
raise InvalidPayloadException(exc.message)
|
|
|
|
payload = JSONPathDict(gl_payload)
|
|
|
|
if payload['object_kind'] != 'push' and payload['object_kind'] != 'tag_push':
|
|
# Unknown kind of webhook.
|
|
raise SkipRequestException
|
|
|
|
# Check for empty commits. The commits list will be empty if the branch is deleted.
|
|
commits = payload['commits']
|
|
if payload['object_kind'] == 'push' and not commits:
|
|
raise SkipRequestException
|
|
|
|
config = SafeDictSetter()
|
|
config['commit'] = payload['checkout_sha']
|
|
config['ref'] = payload['ref']
|
|
config['default_branch'] = default_branch
|
|
config['git_url'] = payload['repository.git_ssh_url']
|
|
|
|
found_commit = JSONPathDict({})
|
|
if payload['object_kind'] == 'push':
|
|
# Find the commit associated with the checkout_sha. Gitlab doesn't (necessary) send this in
|
|
# any order, so we cannot simply index into the commits list.
|
|
found_commit = None
|
|
for commit in commits:
|
|
if commit['id'] == payload['checkout_sha']:
|
|
found_commit = JSONPathDict(commit)
|
|
break
|
|
|
|
if found_commit is None:
|
|
raise SkipRequestException
|
|
|
|
elif payload['object_kind'] == 'tag_push':
|
|
# Gitlab doesn't send commit information for tag pushes (WHY?!), so we need to lookup the
|
|
# commit SHA directly.
|
|
if lookup_commit:
|
|
found_commit_info = lookup_commit(payload['project_id'], payload['checkout_sha'])
|
|
found_commit = JSONPathDict(found_commit_info or {})
|
|
|
|
config['commit_info.url'] = found_commit['url']
|
|
config['commit_info.message'] = found_commit['message']
|
|
config['commit_info.date'] = found_commit['timestamp']
|
|
|
|
# Note: Gitlab does not send full user information with the payload, so we have to
|
|
# (optionally) look it up.
|
|
author_email = found_commit['author.email'] or found_commit['author_email']
|
|
if lookup_user and author_email:
|
|
author_info = lookup_user(author_email)
|
|
if author_info:
|
|
config['commit_info.author.username'] = author_info['username']
|
|
config['commit_info.author.url'] = author_info['html_url']
|
|
config['commit_info.author.avatar_url'] = author_info['avatar_url']
|
|
|
|
return config.dict_value()
|
|
|
|
|
|
class GitLabBuildTrigger(BuildTriggerHandler):
|
|
"""
|
|
BuildTrigger for GitLab.
|
|
"""
|
|
@classmethod
|
|
def service_name(cls):
|
|
return 'gitlab'
|
|
|
|
def _get_authorized_client(self):
|
|
auth_token = self.auth_token or 'invalid'
|
|
return gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=5)
|
|
|
|
def is_active(self):
|
|
return 'hook_id' in self.config
|
|
|
|
@_catch_timeouts
|
|
def activate(self, standard_webhook_url):
|
|
config = self.config
|
|
new_build_source = config['build_source']
|
|
gl_client = self._get_authorized_client()
|
|
|
|
# Find the GitLab repository.
|
|
repository = gl_client.getproject(new_build_source)
|
|
if repository is False:
|
|
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
|
raise TriggerActivationException(msg)
|
|
|
|
# Add a deploy key to the repository.
|
|
public_key, private_key = generate_ssh_keypair()
|
|
config['credentials'] = [
|
|
{
|
|
'name': 'SSH Public Key',
|
|
'value': public_key,
|
|
},
|
|
]
|
|
key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'],
|
|
public_key)
|
|
if key is False:
|
|
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
|
raise TriggerActivationException(msg)
|
|
config['key_id'] = key['id']
|
|
|
|
# Add the webhook to the GitLab repository.
|
|
hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True)
|
|
if hook is False:
|
|
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
|
raise TriggerActivationException(msg)
|
|
|
|
config['hook_id'] = hook['id']
|
|
self.config = config
|
|
return config, {'private_key': private_key}
|
|
|
|
def deactivate(self):
|
|
config = self.config
|
|
gl_client = self._get_authorized_client()
|
|
|
|
# Find the GitLab repository.
|
|
repository = gl_client.getproject(config['build_source'])
|
|
if repository is False:
|
|
msg = 'Unable to find GitLab repository for source: %s' % config['build_source']
|
|
raise TriggerDeactivationException(msg)
|
|
|
|
# Remove the webhook.
|
|
success = gl_client.deleteprojecthook(repository['id'], config['hook_id'])
|
|
if success is False:
|
|
msg = 'Unable to remove hook: %s' % config['hook_id']
|
|
raise TriggerDeactivationException(msg)
|
|
config.pop('hook_id', None)
|
|
|
|
# Remove the key
|
|
success = gl_client.deletedeploykey(repository['id'], config['key_id'])
|
|
if success is False:
|
|
msg = 'Unable to remove deploy key: %s' % config['key_id']
|
|
raise TriggerDeactivationException(msg)
|
|
config.pop('key_id', None)
|
|
|
|
self.config = config
|
|
return config
|
|
|
|
@_catch_timeouts
|
|
def list_build_source_namespaces(self):
|
|
gl_client = self._get_authorized_client()
|
|
current_user = gl_client.currentuser()
|
|
if current_user is False:
|
|
raise RepositoryReadException('Unable to get current user')
|
|
|
|
namespaces = {}
|
|
repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
|
|
for repo in repositories:
|
|
namespace = repo.get('namespace') or {}
|
|
if not namespace:
|
|
continue
|
|
|
|
namespace_id = namespace['id']
|
|
|
|
avatar_url = ''
|
|
if 'avatar' in namespace:
|
|
avatar_data = namespace.get('avatar') or {}
|
|
avatar_url = avatar_data.get('url')
|
|
elif 'owner' in repo:
|
|
owner_data = repo.get('owner') or {}
|
|
avatar_url = owner_data.get('avatar_url')
|
|
|
|
if namespace_id in namespaces:
|
|
namespaces[namespace_id]['score'] = namespaces[namespace_id]['score'] + 1
|
|
else:
|
|
owner = namespace['name']
|
|
namespaces[namespace_id] = {
|
|
'personal': owner == current_user['username'],
|
|
'id': namespace['path'],
|
|
'title': namespace['name'],
|
|
'avatar_url': avatar_url,
|
|
'score': 1,
|
|
'url': gl_client.host + '/' + namespace['path'],
|
|
}
|
|
|
|
return BuildTriggerHandler.build_namespaces_response(namespaces)
|
|
|
|
@_catch_timeouts
|
|
def list_build_sources_for_namespace(self, namespace):
|
|
def repo_view(repo):
|
|
# Because *anything* can be None in GitLab API!
|
|
permissions = repo.get('permissions') or {}
|
|
group_access = permissions.get('group_access') or {}
|
|
project_access = permissions.get('project_access') or {}
|
|
|
|
missing_group_access = permissions.get('group_access') is None
|
|
missing_project_access = permissions.get('project_access') is None
|
|
|
|
access_level = max(group_access.get('access_level') or 0,
|
|
project_access.get('access_level') or 0)
|
|
|
|
has_admin_permission = _ACCESS_LEVEL_MAP.get(access_level, ("", False))[1]
|
|
if missing_group_access or missing_project_access:
|
|
# Default to has permission if we cannot check the permissions. This will allow our users
|
|
# to select the repository and then GitLab's own checks will ensure that the webhook is
|
|
# added only if allowed.
|
|
# TODO: Do we want to display this differently in the UI?
|
|
has_admin_permission = True
|
|
|
|
view = {
|
|
'name': repo['path'],
|
|
'full_name': repo['path_with_namespace'],
|
|
'description': repo.get('description') or '',
|
|
'url': repo.get('web_url'),
|
|
'has_admin_permissions': has_admin_permission,
|
|
'private': repo.get('public', False) is False,
|
|
}
|
|
|
|
if repo.get('last_activity_at'):
|
|
try:
|
|
last_modified = dateutil.parser.parse(repo['last_activity_at'])
|
|
view['last_updated'] = timegm(last_modified.utctimetuple())
|
|
except ValueError:
|
|
logger.exception('Gitlab gave us an invalid last_activity_at: %s', last_modified)
|
|
|
|
return view
|
|
|
|
gl_client = self._get_authorized_client()
|
|
repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
|
|
repos = [repo_view(repo) for repo in repositories if repo['namespace']['path'] == namespace]
|
|
return BuildTriggerHandler.build_sources_response(repos)
|
|
|
|
@_catch_timeouts
|
|
def list_build_subdirs(self):
|
|
config = self.config
|
|
gl_client = self._get_authorized_client()
|
|
new_build_source = config['build_source']
|
|
|
|
repository = gl_client.getproject(new_build_source)
|
|
if repository is False:
|
|
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
|
raise RepositoryReadException(msg)
|
|
|
|
repo_branches = gl_client.getbranches(repository['id'])
|
|
if repo_branches is False:
|
|
msg = 'Unable to find GitLab branches for source: %s' % new_build_source
|
|
raise RepositoryReadException(msg)
|
|
|
|
branches = [branch['name'] for branch in repo_branches]
|
|
branches = find_matching_branches(config, branches)
|
|
branches = branches or [repository['default_branch'] or 'master']
|
|
|
|
repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0])
|
|
if repo_tree is False:
|
|
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
|
|
raise RepositoryReadException(msg)
|
|
|
|
return ["/" + node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
|
|
|
|
@_catch_timeouts
|
|
def load_dockerfile_contents(self):
|
|
gl_client = self._get_authorized_client()
|
|
path = self.get_dockerfile_path()
|
|
|
|
repository = gl_client.getproject(self.config['build_source'])
|
|
if repository is False:
|
|
return None
|
|
|
|
branches = self.list_field_values('branch_name')
|
|
branches = find_matching_branches(self.config, branches)
|
|
if branches == []:
|
|
return None
|
|
|
|
branch_name = branches[0]
|
|
if repository['default_branch'] in branches:
|
|
branch_name = repository['default_branch']
|
|
|
|
contents = gl_client.getrawfile(repository['id'], branch_name, path)
|
|
if contents is False:
|
|
return None
|
|
|
|
return contents
|
|
|
|
@_catch_timeouts
|
|
def list_field_values(self, field_name, limit=None):
|
|
if field_name == 'refs':
|
|
branches = self.list_field_values('branch_name')
|
|
tags = self.list_field_values('tag_name')
|
|
|
|
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
|
[{'kind': 'tag', 'name': t} for t in tags])
|
|
|
|
gl_client = self._get_authorized_client()
|
|
repo = gl_client.getproject(self.config['build_source'])
|
|
if repo is False:
|
|
return []
|
|
|
|
if field_name == 'tag_name':
|
|
tags = gl_client.getrepositorytags(repo['id'])
|
|
if tags is False:
|
|
return []
|
|
|
|
if limit:
|
|
tags = tags[0:limit]
|
|
|
|
return [tag['name'] for tag in tags]
|
|
|
|
if field_name == 'branch_name':
|
|
branches = gl_client.getbranches(repo['id'])
|
|
if branches is False:
|
|
return []
|
|
|
|
if limit:
|
|
branches = branches[0:limit]
|
|
|
|
return [branch['name'] for branch in branches]
|
|
|
|
return None
|
|
|
|
def get_repository_url(self):
|
|
return gitlab_trigger.get_public_url(self.config['build_source'])
|
|
|
|
@_catch_timeouts
|
|
def lookup_commit(self, repo_id, commit_sha):
|
|
if repo_id is None:
|
|
return None
|
|
|
|
gl_client = self._get_authorized_client()
|
|
commit = gl_client.getrepositorycommit(repo_id, commit_sha)
|
|
if commit is False:
|
|
return None
|
|
|
|
return commit
|
|
|
|
@_catch_timeouts
|
|
def lookup_user(self, email):
|
|
gl_client = self._get_authorized_client()
|
|
try:
|
|
result = gl_client.getusers(search=email)
|
|
if result is False:
|
|
return None
|
|
|
|
[user] = result
|
|
return {
|
|
'username': user['username'],
|
|
'html_url': gl_client.host + '/' + user['username'],
|
|
'avatar_url': user['avatar_url']
|
|
}
|
|
except ValueError:
|
|
return None
|
|
|
|
@_catch_timeouts
|
|
def get_metadata_for_commit(self, commit_sha, ref, repo):
|
|
gl_client = self._get_authorized_client()
|
|
commit = gl_client.getrepositorycommit(repo['id'], commit_sha)
|
|
|
|
metadata = {
|
|
'commit': commit['id'],
|
|
'ref': ref,
|
|
'default_branch': repo['default_branch'],
|
|
'git_url': repo['ssh_url_to_repo'],
|
|
'commit_info': {
|
|
'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'],
|
|
'message': commit['message'],
|
|
'date': commit['committed_date'],
|
|
},
|
|
}
|
|
|
|
committer = None
|
|
if 'committer_email' in commit:
|
|
committer = self.lookup_user(commit['committer_email'])
|
|
|
|
author = None
|
|
if 'author_email' in commit:
|
|
author = self.lookup_user(commit['author_email'])
|
|
|
|
if committer is not None:
|
|
metadata['commit_info']['committer'] = {
|
|
'username': committer['username'],
|
|
'avatar_url': committer['avatar_url'],
|
|
'url': gl_client.host + '/' + committer['username'],
|
|
}
|
|
|
|
if author is not None:
|
|
metadata['commit_info']['author'] = {
|
|
'username': author['username'],
|
|
'avatar_url': author['avatar_url'],
|
|
'url': gl_client.host + '/' + author['username']
|
|
}
|
|
|
|
return metadata
|
|
|
|
@_catch_timeouts
|
|
def manual_start(self, run_parameters=None):
|
|
gl_client = self._get_authorized_client()
|
|
|
|
repo = gl_client.getproject(self.config['build_source'])
|
|
if repo is False:
|
|
raise TriggerStartException('Could not find repository')
|
|
|
|
def get_tag_sha(tag_name):
|
|
tags = gl_client.getrepositorytags(repo['id'])
|
|
if tags is False:
|
|
raise TriggerStartException('Could not find tag in repository')
|
|
|
|
for tag in tags:
|
|
if tag['name'] == tag_name:
|
|
return tag['commit']['id']
|
|
|
|
raise TriggerStartException('Could not find tag in repository')
|
|
|
|
def get_branch_sha(branch_name):
|
|
branch = gl_client.getbranch(repo['id'], branch_name)
|
|
if branch is False:
|
|
raise TriggerStartException('Could not find branch in repository')
|
|
|
|
return branch['commit']['id']
|
|
|
|
# Find the branch or tag to build.
|
|
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
|
|
repo['default_branch'])
|
|
|
|
metadata = self.get_metadata_for_commit(commit_sha, ref, repo)
|
|
return self.prepare_build(metadata, is_manual=True)
|
|
|
|
@_catch_timeouts
|
|
def handle_trigger_request(self, request):
|
|
payload = request.get_json()
|
|
if not payload:
|
|
raise InvalidPayloadException()
|
|
|
|
logger.debug('GitLab trigger payload %s', payload)
|
|
|
|
# Lookup the default branch.
|
|
gl_client = self._get_authorized_client()
|
|
repo = gl_client.getproject(self.config['build_source'])
|
|
if repo is False:
|
|
logger.debug('Skipping GitLab build; project %s not found', self.config['build_source'])
|
|
raise InvalidPayloadException()
|
|
|
|
default_branch = repo['default_branch']
|
|
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
|
|
lookup_user=self.lookup_user,
|
|
lookup_commit=self.lookup_commit)
|
|
prepared = self.prepare_build(metadata)
|
|
|
|
# Check if we should skip this build.
|
|
raise_if_skipped_build(prepared, self.config)
|
|
return prepared
|