594 lines
19 KiB
Python
594 lines
19 KiB
Python
import os.path
|
|
import logging
|
|
|
|
from calendar import timegm
|
|
from functools import wraps
|
|
|
|
import dateutil.parser
|
|
import gitlab
|
|
import requests
|
|
|
|
from jsonschema import validate
|
|
|
|
from app import app, gitlab_trigger
|
|
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
|
TriggerDeactivationException, TriggerStartException,
|
|
SkipRequestException, InvalidPayloadException,
|
|
TriggerAuthException,
|
|
determine_build_ref, raise_if_skipped_build,
|
|
find_matching_branches)
|
|
from buildtrigger.basehandler import BuildTriggerHandler
|
|
from endpoints.exception import ExternalServiceError
|
|
from util.security.ssh import generate_ssh_keypair
|
|
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
GITLAB_WEBHOOK_PAYLOAD_SCHEMA = {
|
|
'type': 'object',
|
|
'properties': {
|
|
'ref': {
|
|
'type': 'string',
|
|
},
|
|
'checkout_sha': {
|
|
'type': ['string', 'null'],
|
|
},
|
|
'repository': {
|
|
'type': 'object',
|
|
'properties': {
|
|
'git_ssh_url': {
|
|
'type': 'string',
|
|
},
|
|
},
|
|
'required': ['git_ssh_url'],
|
|
},
|
|
'commits': {
|
|
'type': 'array',
|
|
'items': {
|
|
'type': 'object',
|
|
'properties': {
|
|
'id': {
|
|
'type': 'string',
|
|
},
|
|
'url': {
|
|
'type': 'string',
|
|
},
|
|
'message': {
|
|
'type': 'string',
|
|
},
|
|
'timestamp': {
|
|
'type': 'string',
|
|
},
|
|
'author': {
|
|
'type': 'object',
|
|
'properties': {
|
|
'email': {
|
|
'type': 'string',
|
|
},
|
|
},
|
|
'required': ['email'],
|
|
},
|
|
},
|
|
'required': ['id', 'url', 'message', 'timestamp'],
|
|
},
|
|
},
|
|
},
|
|
'required': ['ref', 'checkout_sha', 'repository'],
|
|
}
|
|
|
|
_ACCESS_LEVEL_MAP = {
|
|
50: ("owner", True),
|
|
40: ("master", True),
|
|
30: ("developer", False),
|
|
20: ("reporter", False),
|
|
10: ("guest", False),
|
|
}
|
|
|
|
_PER_PAGE_COUNT = 20
|
|
|
|
|
|
def _catch_timeouts_and_errors(func):
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except requests.exceptions.Timeout:
|
|
msg = 'Request to the GitLab API timed out'
|
|
logger.exception(msg)
|
|
raise ExternalServiceError(msg)
|
|
except gitlab.GitlabError:
|
|
msg = 'GitLab API error. Please contact support.'
|
|
logger.exception(msg)
|
|
raise ExternalServiceError(msg)
|
|
return wrapper
|
|
|
|
|
|
def _paginated_iterator(func, exc, **kwargs):
|
|
""" Returns an iterator over invocations of the given function, automatically handling
|
|
pagination.
|
|
"""
|
|
page = 1
|
|
while True:
|
|
result = func(page=page, per_page=_PER_PAGE_COUNT, **kwargs)
|
|
if result is None or result is False:
|
|
raise exc
|
|
|
|
counter = 0
|
|
for item in result:
|
|
yield item
|
|
counter = counter + 1
|
|
|
|
if counter < _PER_PAGE_COUNT:
|
|
break
|
|
|
|
page = page + 1
|
|
|
|
|
|
def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None,
|
|
lookup_commit=None):
|
|
""" Returns the Gitlab webhook JSON payload transformed into our own payload
|
|
format. If the gl_payload is not valid, returns None.
|
|
"""
|
|
try:
|
|
validate(gl_payload, GITLAB_WEBHOOK_PAYLOAD_SCHEMA)
|
|
except Exception as exc:
|
|
raise InvalidPayloadException(exc.message)
|
|
|
|
payload = JSONPathDict(gl_payload)
|
|
|
|
if payload['object_kind'] != 'push' and payload['object_kind'] != 'tag_push':
|
|
# Unknown kind of webhook.
|
|
raise SkipRequestException
|
|
|
|
# Check for empty commits. The commits list will be empty if the branch is deleted.
|
|
commits = payload['commits']
|
|
if payload['object_kind'] == 'push' and not commits:
|
|
raise SkipRequestException
|
|
|
|
config = SafeDictSetter()
|
|
config['commit'] = payload['checkout_sha']
|
|
config['ref'] = payload['ref']
|
|
config['default_branch'] = default_branch
|
|
config['git_url'] = payload['repository.git_ssh_url']
|
|
|
|
found_commit = JSONPathDict({})
|
|
if payload['object_kind'] == 'push':
|
|
# Find the commit associated with the checkout_sha. Gitlab doesn't (necessary) send this in
|
|
# any order, so we cannot simply index into the commits list.
|
|
found_commit = None
|
|
for commit in commits:
|
|
if commit['id'] == payload['checkout_sha']:
|
|
found_commit = JSONPathDict(commit)
|
|
break
|
|
|
|
if found_commit is None:
|
|
raise SkipRequestException
|
|
|
|
elif payload['object_kind'] == 'tag_push':
|
|
# Gitlab doesn't send commit information for tag pushes (WHY?!), so we need to lookup the
|
|
# commit SHA directly.
|
|
if lookup_commit:
|
|
found_commit_info = lookup_commit(payload['project_id'], payload['checkout_sha'])
|
|
found_commit = JSONPathDict(found_commit_info or {})
|
|
|
|
config['commit_info.url'] = found_commit['url']
|
|
config['commit_info.message'] = found_commit['message']
|
|
config['commit_info.date'] = found_commit['timestamp']
|
|
|
|
# Note: Gitlab does not send full user information with the payload, so we have to
|
|
# (optionally) look it up.
|
|
author_email = found_commit['author.email'] or found_commit['author_email']
|
|
if lookup_user and author_email:
|
|
author_info = lookup_user(author_email)
|
|
if author_info:
|
|
config['commit_info.author.username'] = author_info['username']
|
|
config['commit_info.author.url'] = author_info['html_url']
|
|
config['commit_info.author.avatar_url'] = author_info['avatar_url']
|
|
|
|
return config.dict_value()
|
|
|
|
|
|
class GitLabBuildTrigger(BuildTriggerHandler):
|
|
"""
|
|
BuildTrigger for GitLab.
|
|
"""
|
|
@classmethod
|
|
def service_name(cls):
|
|
return 'gitlab'
|
|
|
|
def _get_authorized_client(self):
|
|
auth_token = self.auth_token or 'invalid'
|
|
api_version = self.config.get('API_VERSION', '4')
|
|
client = gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=20,
|
|
api_version=api_version)
|
|
try:
|
|
client.auth()
|
|
except gitlab.GitlabGetError as ex:
|
|
raise TriggerAuthException(ex.message)
|
|
|
|
return client
|
|
|
|
def is_active(self):
|
|
return 'hook_id' in self.config
|
|
|
|
@_catch_timeouts_and_errors
|
|
def activate(self, standard_webhook_url):
|
|
config = self.config
|
|
new_build_source = config['build_source']
|
|
gl_client = self._get_authorized_client()
|
|
|
|
# Find the GitLab repository.
|
|
gl_project = gl_client.projects.get(new_build_source)
|
|
if not gl_project:
|
|
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
|
raise TriggerActivationException(msg)
|
|
|
|
# Add a deploy key to the repository.
|
|
public_key, private_key = generate_ssh_keypair()
|
|
config['credentials'] = [
|
|
{
|
|
'name': 'SSH Public Key',
|
|
'value': public_key,
|
|
},
|
|
]
|
|
|
|
key = gl_project.keys.create({
|
|
'title': '%s Builder' % app.config['REGISTRY_TITLE'],
|
|
'key': public_key,
|
|
})
|
|
|
|
if not key:
|
|
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
|
raise TriggerActivationException(msg)
|
|
|
|
config['key_id'] = key.get_id()
|
|
|
|
# Add the webhook to the GitLab repository.
|
|
hook = gl_project.hooks.create({
|
|
'url': standard_webhook_url,
|
|
'push': True,
|
|
})
|
|
if not hook:
|
|
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
|
raise TriggerActivationException(msg)
|
|
|
|
config['hook_id'] = hook.get_id()
|
|
self.config = config
|
|
return config, {'private_key': private_key}
|
|
|
|
def deactivate(self):
|
|
config = self.config
|
|
gl_client = self._get_authorized_client()
|
|
|
|
# Find the GitLab repository.
|
|
gl_project = gl_client.projects.get(config['build_source'])
|
|
if not gl_project:
|
|
msg = 'Unable to find GitLab repository for source: %s' % config['build_source']
|
|
raise TriggerDeactivationException(msg)
|
|
|
|
# Remove the webhook.
|
|
gl_project.hooks.delete(config['hook_id'])
|
|
config.pop('hook_id', None)
|
|
|
|
# Remove the key
|
|
gl_project.keys.delete(config['key_id'])
|
|
config.pop('key_id', None)
|
|
|
|
self.config = config
|
|
return config
|
|
|
|
@_catch_timeouts_and_errors
|
|
def list_build_source_namespaces(self):
|
|
gl_client = self._get_authorized_client()
|
|
current_user = gl_client.user
|
|
if not current_user:
|
|
raise RepositoryReadException('Unable to get current user')
|
|
|
|
namespaces = {}
|
|
for namespace in _paginated_iterator(gl_client.namespaces.list, RepositoryReadException):
|
|
namespace_id = namespace.get_id()
|
|
|
|
# Retrieve the namespace as a user or group.
|
|
namespace_obj = self._get_namespace(gl_client, namespace)
|
|
if namespace_obj is None:
|
|
logger.warning('Could not load details for namespace %s', namespace_id)
|
|
continue
|
|
|
|
if namespace_id in namespaces:
|
|
namespaces[namespace_id]['score'] = namespaces[namespace_id]['score'] + 1
|
|
else:
|
|
owner = namespace.attributes['name']
|
|
namespaces[namespace_id] = {
|
|
'personal': owner == current_user.attributes['username'],
|
|
'id': str(namespace_id),
|
|
'title': namespace.attributes['name'],
|
|
'avatar_url': namespace_obj.attributes.get('avatar_url', ''),
|
|
'score': 1,
|
|
'url': namespace_obj.attributes.get('web_url', ''),
|
|
}
|
|
|
|
return BuildTriggerHandler.build_namespaces_response(namespaces)
|
|
|
|
def _get_namespace(self, gl_client, gl_namespace, lazy=False):
|
|
try:
|
|
if gl_namespace.attributes['kind'] == 'group':
|
|
return gl_client.groups.get(gl_namespace.attributes['id'], lazy=lazy)
|
|
|
|
if gl_namespace.attributes['name'] == gl_client.user.attributes['username']:
|
|
return gl_client.users.get(gl_client.user.attributes['id'], lazy=lazy)
|
|
|
|
# Note: This doesn't seem to work for IDs retrieved via the namespaces API; the IDs are
|
|
# different.
|
|
return gl_client.users.get(gl_namespace.attributes['id'], lazy=lazy)
|
|
except gitlab.GitlabGetError:
|
|
return None
|
|
|
|
@_catch_timeouts_and_errors
|
|
def list_build_sources_for_namespace(self, namespace_id):
|
|
if not namespace_id:
|
|
return []
|
|
|
|
def repo_view(repo):
|
|
# Because *anything* can be None in GitLab API!
|
|
permissions = repo.attributes.get('permissions') or {}
|
|
group_access = permissions.get('group_access') or {}
|
|
project_access = permissions.get('project_access') or {}
|
|
|
|
missing_group_access = permissions.get('group_access') is None
|
|
missing_project_access = permissions.get('project_access') is None
|
|
|
|
access_level = max(group_access.get('access_level') or 0,
|
|
project_access.get('access_level') or 0)
|
|
|
|
has_admin_permission = _ACCESS_LEVEL_MAP.get(access_level, ("", False))[1]
|
|
if missing_group_access or missing_project_access:
|
|
# Default to has permission if we cannot check the permissions. This will allow our users
|
|
# to select the repository and then GitLab's own checks will ensure that the webhook is
|
|
# added only if allowed.
|
|
# TODO: Do we want to display this differently in the UI?
|
|
has_admin_permission = True
|
|
|
|
view = {
|
|
'name': repo.attributes['path'],
|
|
'full_name': repo.attributes['path_with_namespace'],
|
|
'description': repo.attributes.get('description') or '',
|
|
'url': repo.attributes.get('web_url'),
|
|
'has_admin_permissions': has_admin_permission,
|
|
'private': repo.attributes.get('visibility') == 'private',
|
|
}
|
|
|
|
if repo.attributes.get('last_activity_at'):
|
|
try:
|
|
last_modified = dateutil.parser.parse(repo.attributes['last_activity_at'])
|
|
view['last_updated'] = timegm(last_modified.utctimetuple())
|
|
except ValueError:
|
|
logger.exception('Gitlab gave us an invalid last_activity_at: %s', last_modified)
|
|
|
|
return view
|
|
|
|
gl_client = self._get_authorized_client()
|
|
|
|
try:
|
|
gl_namespace = gl_client.namespaces.get(namespace_id)
|
|
except gitlab.GitlabGetError:
|
|
return []
|
|
|
|
namespace_obj = self._get_namespace(gl_client, gl_namespace, lazy=True)
|
|
repositories = _paginated_iterator(namespace_obj.projects.list, RepositoryReadException)
|
|
return BuildTriggerHandler.build_sources_response([repo_view(repo) for repo in repositories])
|
|
|
|
@_catch_timeouts_and_errors
|
|
def list_build_subdirs(self):
|
|
config = self.config
|
|
gl_client = self._get_authorized_client()
|
|
new_build_source = config['build_source']
|
|
|
|
gl_project = gl_client.projects.get(new_build_source)
|
|
if not gl_project:
|
|
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
|
raise RepositoryReadException(msg)
|
|
|
|
repo_branches = gl_project.branches.list()
|
|
if not repo_branches:
|
|
msg = 'Unable to find GitLab branches for source: %s' % new_build_source
|
|
raise RepositoryReadException(msg)
|
|
|
|
branches = [branch.attributes['name'] for branch in repo_branches]
|
|
branches = find_matching_branches(config, branches)
|
|
branches = branches or [gl_project.attributes['default_branch'] or 'master']
|
|
|
|
repo_tree = gl_project.repository_tree(ref=branches[0])
|
|
if not repo_tree:
|
|
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
|
|
raise RepositoryReadException(msg)
|
|
|
|
return [node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
|
|
|
|
@_catch_timeouts_and_errors
|
|
def load_dockerfile_contents(self):
|
|
gl_client = self._get_authorized_client()
|
|
path = self.get_dockerfile_path()
|
|
|
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
|
if not gl_project:
|
|
return None
|
|
|
|
branches = self.list_field_values('branch_name')
|
|
branches = find_matching_branches(self.config, branches)
|
|
if branches == []:
|
|
return None
|
|
|
|
branch_name = branches[0]
|
|
if gl_project.attributes['default_branch'] in branches:
|
|
branch_name = gl_project.attributes['default_branch']
|
|
|
|
try:
|
|
return gl_project.files.get(path, branch_name).decode()
|
|
except gitlab.GitlabGetError:
|
|
return None
|
|
|
|
@_catch_timeouts_and_errors
|
|
def list_field_values(self, field_name, limit=None):
|
|
if field_name == 'refs':
|
|
branches = self.list_field_values('branch_name')
|
|
tags = self.list_field_values('tag_name')
|
|
|
|
return ([{'kind': 'branch', 'name': b} for b in branches] +
|
|
[{'kind': 'tag', 'name': t} for t in tags])
|
|
|
|
gl_client = self._get_authorized_client()
|
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
|
if not gl_project:
|
|
return []
|
|
|
|
if field_name == 'tag_name':
|
|
tags = gl_project.tags.list()
|
|
if not tags:
|
|
return []
|
|
|
|
if limit:
|
|
tags = tags[0:limit]
|
|
|
|
return [tag.attributes['name'] for tag in tags]
|
|
|
|
if field_name == 'branch_name':
|
|
branches = gl_project.branches.list()
|
|
if not branches:
|
|
return []
|
|
|
|
if limit:
|
|
branches = branches[0:limit]
|
|
|
|
return [branch.attributes['name'] for branch in branches]
|
|
|
|
return None
|
|
|
|
def get_repository_url(self):
|
|
return gitlab_trigger.get_public_url(self.config['build_source'])
|
|
|
|
@_catch_timeouts_and_errors
|
|
def lookup_commit(self, repo_id, commit_sha):
|
|
if repo_id is None:
|
|
return None
|
|
|
|
gl_client = self._get_authorized_client()
|
|
gl_project = gl_client.projects.get(self.config['build_source'], lazy=True)
|
|
commit = gl_project.commits.get(commit_sha)
|
|
if not commit:
|
|
return None
|
|
|
|
return commit
|
|
|
|
@_catch_timeouts_and_errors
|
|
def lookup_user(self, email):
|
|
gl_client = self._get_authorized_client()
|
|
try:
|
|
result = gl_client.users.list(search=email)
|
|
if not result:
|
|
return None
|
|
|
|
[user] = result
|
|
return {
|
|
'username': user.attributes['username'],
|
|
'html_url': user.attributes['web_url'],
|
|
'avatar_url': user.attributes['avatar_url']
|
|
}
|
|
except ValueError:
|
|
return None
|
|
|
|
@_catch_timeouts_and_errors
|
|
def get_metadata_for_commit(self, commit_sha, ref, repo):
|
|
commit = self.lookup_commit(repo.get_id(), commit_sha)
|
|
if commit is None:
|
|
return None
|
|
|
|
metadata = {
|
|
'commit': commit.attributes['id'],
|
|
'ref': ref,
|
|
'default_branch': repo.attributes['default_branch'],
|
|
'git_url': repo.attributes['ssh_url_to_repo'],
|
|
'commit_info': {
|
|
'url': os.path.join(repo.attributes['web_url'], 'commit', commit.attributes['id']),
|
|
'message': commit.attributes['message'],
|
|
'date': commit.attributes['committed_date'],
|
|
},
|
|
}
|
|
|
|
committer = None
|
|
if 'committer_email' in commit.attributes:
|
|
committer = self.lookup_user(commit.attributes['committer_email'])
|
|
|
|
author = None
|
|
if 'author_email' in commit.attributes:
|
|
author = self.lookup_user(commit.attributes['author_email'])
|
|
|
|
if committer is not None:
|
|
metadata['commit_info']['committer'] = {
|
|
'username': committer['username'],
|
|
'avatar_url': committer['avatar_url'],
|
|
'url': committer.get('http_url', ''),
|
|
}
|
|
|
|
if author is not None:
|
|
metadata['commit_info']['author'] = {
|
|
'username': author['username'],
|
|
'avatar_url': author['avatar_url'],
|
|
'url': author.get('http_url', ''),
|
|
}
|
|
|
|
return metadata
|
|
|
|
@_catch_timeouts_and_errors
|
|
def manual_start(self, run_parameters=None):
|
|
gl_client = self._get_authorized_client()
|
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
|
if not gl_project:
|
|
raise TriggerStartException('Could not find repository')
|
|
|
|
def get_tag_sha(tag_name):
|
|
try:
|
|
tag = gl_project.tags.get(tag_name)
|
|
except gitlab.GitlabGetError:
|
|
raise TriggerStartException('Could not find tag in repository')
|
|
|
|
return tag.attributes['commit']['id']
|
|
|
|
def get_branch_sha(branch_name):
|
|
try:
|
|
branch = gl_project.branches.get(branch_name)
|
|
except gitlab.GitlabGetError:
|
|
raise TriggerStartException('Could not find branch in repository')
|
|
|
|
return branch.attributes['commit']['id']
|
|
|
|
# Find the branch or tag to build.
|
|
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
|
|
gl_project.attributes['default_branch'])
|
|
|
|
metadata = self.get_metadata_for_commit(commit_sha, ref, gl_project)
|
|
return self.prepare_build(metadata, is_manual=True)
|
|
|
|
@_catch_timeouts_and_errors
|
|
def handle_trigger_request(self, request):
|
|
payload = request.get_json()
|
|
if not payload:
|
|
raise InvalidPayloadException()
|
|
|
|
logger.debug('GitLab trigger payload %s', payload)
|
|
|
|
# Lookup the default branch.
|
|
gl_client = self._get_authorized_client()
|
|
gl_project = gl_client.projects.get(self.config['build_source'])
|
|
if not gl_project:
|
|
logger.debug('Skipping GitLab build; project %s not found', self.config['build_source'])
|
|
raise InvalidPayloadException()
|
|
|
|
default_branch = gl_project.attributes['default_branch']
|
|
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
|
|
lookup_user=self.lookup_user,
|
|
lookup_commit=self.lookup_commit)
|
|
prepared = self.prepare_build(metadata)
|
|
|
|
# Check if we should skip this build.
|
|
raise_if_skipped_build(prepared, self.config)
|
|
return prepared
|