Reimplement GitLab trigger handler using the V4 API library

GitLab has deprecated and removed the V3 API entirely

Fixes https://jira.coreos.com/browse/QUAY-966
This commit is contained in:
Joseph Schorr 2018-06-11 15:22:02 -04:00
parent b0489aa8b0
commit bf966545ba
9 changed files with 912 additions and 393 deletions

View file

@ -1,28 +1,26 @@
import os.path
import logging
import os
from calendar import timegm
from functools import wraps
import dateutil.parser
from app import app, gitlab_trigger
import gitlab
import requests
from jsonschema import validate
from app import app, gitlab_trigger
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
TriggerDeactivationException, TriggerStartException,
SkipRequestException, InvalidPayloadException,
TriggerAuthException,
determine_build_ref, raise_if_skipped_build,
find_matching_branches)
from buildtrigger.basehandler import BuildTriggerHandler
from endpoints.exception import ExternalServiceError
from util.security.ssh import generate_ssh_keypair
from util.dict_wrappers import JSONPathDict, SafeDictSetter
from endpoints.exception import ExternalServiceError
import gitlab
import requests
logger = logging.getLogger(__name__)
@ -89,7 +87,7 @@ _ACCESS_LEVEL_MAP = {
_PER_PAGE_COUNT = 20
def _catch_timeouts(func):
def _catch_timeouts_and_errors(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
@ -98,17 +96,21 @@ def _catch_timeouts(func):
msg = 'Request to the GitLab API timed out'
logger.exception(msg)
raise ExternalServiceError(msg)
except gitlab.GitlabError:
msg = 'GitLab API error. Please contact support.'
logger.exception(msg)
raise ExternalServiceError(msg)
return wrapper
def _paginated_iterator(func, exc):
def _paginated_iterator(func, exc, **kwargs):
""" Returns an iterator over invocations of the given function, automatically handling
pagination.
"""
page = 0
page = 1
while True:
result = func(page=page, per_page=_PER_PAGE_COUNT)
if result is False:
result = func(page=page, per_page=_PER_PAGE_COUNT, **kwargs)
if result is None or result is False:
raise exc
counter = 0
@ -196,20 +198,28 @@ class GitLabBuildTrigger(BuildTriggerHandler):
def _get_authorized_client(self):
auth_token = self.auth_token or 'invalid'
return gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=5)
api_version = self.config.get('API_VERSION', '4')
client = gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=20,
api_version=api_version)
try:
client.auth()
except gitlab.GitlabGetError as ex:
raise TriggerAuthException(ex.message)
return client
def is_active(self):
return 'hook_id' in self.config
@_catch_timeouts
@_catch_timeouts_and_errors
def activate(self, standard_webhook_url):
config = self.config
new_build_source = config['build_source']
gl_client = self._get_authorized_client()
# Find the GitLab repository.
repository = gl_client.getproject(new_build_source)
if repository is False:
gl_project = gl_client.projects.get(new_build_source)
if not gl_project:
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
raise TriggerActivationException(msg)
@ -221,20 +231,28 @@ class GitLabBuildTrigger(BuildTriggerHandler):
'value': public_key,
},
]
key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'],
public_key)
if key is False:
key = gl_project.keys.create({
'title': '%s Builder' % app.config['REGISTRY_TITLE'],
'key': public_key,
})
if not key:
msg = 'Unable to add deploy key to repository: %s' % new_build_source
raise TriggerActivationException(msg)
config['key_id'] = key['id']
config['key_id'] = key.get_id()
# Add the webhook to the GitLab repository.
hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True)
if hook is False:
hook = gl_project.hooks.create({
'url': standard_webhook_url,
'push': True,
})
if not hook:
msg = 'Unable to create webhook on repository: %s' % new_build_source
raise TriggerActivationException(msg)
config['hook_id'] = hook['id']
config['hook_id'] = hook.get_id()
self.config = config
return config, {'private_key': private_key}
@ -243,72 +261,71 @@ class GitLabBuildTrigger(BuildTriggerHandler):
gl_client = self._get_authorized_client()
# Find the GitLab repository.
repository = gl_client.getproject(config['build_source'])
if repository is False:
gl_project = gl_client.projects.get(config['build_source'])
if not gl_project:
msg = 'Unable to find GitLab repository for source: %s' % config['build_source']
raise TriggerDeactivationException(msg)
# Remove the webhook.
success = gl_client.deleteprojecthook(repository['id'], config['hook_id'])
if success is False:
msg = 'Unable to remove hook: %s' % config['hook_id']
raise TriggerDeactivationException(msg)
gl_project.hooks.delete(config['hook_id'])
config.pop('hook_id', None)
# Remove the key
success = gl_client.deletedeploykey(repository['id'], config['key_id'])
if success is False:
msg = 'Unable to remove deploy key: %s' % config['key_id']
raise TriggerDeactivationException(msg)
gl_project.keys.delete(config['key_id'])
config.pop('key_id', None)
self.config = config
return config
@_catch_timeouts
@_catch_timeouts_and_errors
def list_build_source_namespaces(self):
gl_client = self._get_authorized_client()
current_user = gl_client.currentuser()
if current_user is False:
current_user = gl_client.user
if not current_user:
raise RepositoryReadException('Unable to get current user')
namespaces = {}
repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
for repo in repositories:
namespace = repo.get('namespace') or {}
if not namespace:
for namespace in _paginated_iterator(gl_client.namespaces.list, RepositoryReadException):
namespace_id = namespace.get_id()
# Retrieve the namespace as a user or group.
namespace_obj = self._get_namespace(gl_client, namespace)
if namespace_obj is None:
logger.warning('Could not load details for namespace %s', namespace_id)
continue
namespace_id = namespace['id']
avatar_url = ''
if 'avatar' in namespace:
avatar_data = namespace.get('avatar') or {}
avatar_url = avatar_data.get('url')
elif 'owner' in repo:
owner_data = repo.get('owner') or {}
avatar_url = owner_data.get('avatar_url')
if namespace_id in namespaces:
namespaces[namespace_id]['score'] = namespaces[namespace_id]['score'] + 1
else:
owner = namespace['name']
owner = namespace.attributes['name']
namespaces[namespace_id] = {
'personal': owner == current_user['username'],
'id': namespace['path'],
'title': namespace['name'],
'avatar_url': avatar_url,
'personal': owner == current_user.attributes['username'],
'id': str(namespace_id),
'title': namespace.attributes['name'],
'avatar_url': namespace_obj.attributes.get('avatar_url', ''),
'score': 1,
'url': gl_client.host + '/' + namespace['path'],
'url': namespace_obj.attributes.get('web_url', ''),
}
return BuildTriggerHandler.build_namespaces_response(namespaces)
@_catch_timeouts
def list_build_sources_for_namespace(self, namespace):
def _get_namespace(self, gl_client, gl_namespace, lazy=False):
try:
if gl_namespace.attributes['kind'] == 'group':
return gl_client.groups.get(gl_namespace.attributes['name'], lazy=lazy)
return gl_client.users.get(gl_namespace.attributes['name'], lazy=lazy)
except gitlab.GitlabGetError:
return None
@_catch_timeouts_and_errors
def list_build_sources_for_namespace(self, namespace_id):
if not namespace_id:
return []
def repo_view(repo):
# Because *anything* can be None in GitLab API!
permissions = repo.get('permissions') or {}
permissions = repo.attributes.get('permissions') or {}
group_access = permissions.get('group_access') or {}
project_access = permissions.get('project_access') or {}
@ -327,17 +344,17 @@ class GitLabBuildTrigger(BuildTriggerHandler):
has_admin_permission = True
view = {
'name': repo['path'],
'full_name': repo['path_with_namespace'],
'description': repo.get('description') or '',
'url': repo.get('web_url'),
'name': repo.attributes['path'],
'full_name': repo.attributes['path_with_namespace'],
'description': repo.attributes.get('description') or '',
'url': repo.attributes.get('web_url'),
'has_admin_permissions': has_admin_permission,
'private': repo.get('public', False) is False,
'private': repo.attributes.get('visibility') == 'private',
}
if repo.get('last_activity_at'):
if repo.attributes.get('last_activity_at'):
try:
last_modified = dateutil.parser.parse(repo['last_activity_at'])
last_modified = dateutil.parser.parse(repo.attributes['last_activity_at'])
view['last_updated'] = timegm(last_modified.utctimetuple())
except ValueError:
logger.exception('Gitlab gave us an invalid last_activity_at: %s', last_modified)
@ -345,44 +362,50 @@ class GitLabBuildTrigger(BuildTriggerHandler):
return view
gl_client = self._get_authorized_client()
repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
repos = [repo_view(repo) for repo in repositories if repo['namespace']['path'] == namespace]
return BuildTriggerHandler.build_sources_response(repos)
@_catch_timeouts
try:
gl_namespace = gl_client.namespaces.get(namespace_id)
except gitlab.GitlabGetError:
return []
namespace_obj = self._get_namespace(gl_client, gl_namespace, lazy=True)
repositories = _paginated_iterator(namespace_obj.projects.list, RepositoryReadException)
return BuildTriggerHandler.build_sources_response([repo_view(repo) for repo in repositories])
@_catch_timeouts_and_errors
def list_build_subdirs(self):
config = self.config
gl_client = self._get_authorized_client()
new_build_source = config['build_source']
repository = gl_client.getproject(new_build_source)
if repository is False:
gl_project = gl_client.projects.get(new_build_source)
if not gl_project:
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
raise RepositoryReadException(msg)
repo_branches = gl_client.getbranches(repository['id'])
if repo_branches is False:
repo_branches = gl_project.branches.list()
if not repo_branches:
msg = 'Unable to find GitLab branches for source: %s' % new_build_source
raise RepositoryReadException(msg)
branches = [branch['name'] for branch in repo_branches]
branches = [branch.attributes['name'] for branch in repo_branches]
branches = find_matching_branches(config, branches)
branches = branches or [repository['default_branch'] or 'master']
branches = branches or [gl_project.attributes['default_branch'] or 'master']
repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0])
if repo_tree is False:
repo_tree = gl_project.repository_tree(ref=branches[0])
if not repo_tree:
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
raise RepositoryReadException(msg)
return ["/" + node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
return [node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
@_catch_timeouts
@_catch_timeouts_and_errors
def load_dockerfile_contents(self):
gl_client = self._get_authorized_client()
path = self.get_dockerfile_path()
repository = gl_client.getproject(self.config['build_source'])
if repository is False:
gl_project = gl_client.projects.get(self.config['build_source'])
if not gl_project:
return None
branches = self.list_field_values('branch_name')
@ -391,16 +414,15 @@ class GitLabBuildTrigger(BuildTriggerHandler):
return None
branch_name = branches[0]
if repository['default_branch'] in branches:
branch_name = repository['default_branch']
if gl_project.attributes['default_branch'] in branches:
branch_name = gl_project.attributes['default_branch']
contents = gl_client.getrawfile(repository['id'], branch_name, path)
if contents is False:
try:
return gl_project.files.get(path, branch_name).decode()
except gitlab.GitlabGetError:
return None
return contents
@_catch_timeouts
@_catch_timeouts_and_errors
def list_field_values(self, field_name, limit=None):
if field_name == 'refs':
branches = self.list_field_values('branch_name')
@ -410,139 +432,138 @@ class GitLabBuildTrigger(BuildTriggerHandler):
[{'kind': 'tag', 'name': t} for t in tags])
gl_client = self._get_authorized_client()
repo = gl_client.getproject(self.config['build_source'])
if repo is False:
gl_project = gl_client.projects.get(self.config['build_source'])
if not gl_project:
return []
if field_name == 'tag_name':
tags = gl_client.getrepositorytags(repo['id'])
if tags is False:
tags = gl_project.tags.list()
if not tags:
return []
if limit:
tags = tags[0:limit]
return [tag['name'] for tag in tags]
return [tag.attributes['name'] for tag in tags]
if field_name == 'branch_name':
branches = gl_client.getbranches(repo['id'])
if branches is False:
branches = gl_project.branches.list()
if not branches:
return []
if limit:
branches = branches[0:limit]
return [branch['name'] for branch in branches]
return [branch.attributes['name'] for branch in branches]
return None
def get_repository_url(self):
return gitlab_trigger.get_public_url(self.config['build_source'])
@_catch_timeouts
@_catch_timeouts_and_errors
def lookup_commit(self, repo_id, commit_sha):
if repo_id is None:
return None
gl_client = self._get_authorized_client()
commit = gl_client.getrepositorycommit(repo_id, commit_sha)
if commit is False:
gl_project = gl_client.projects.get(self.config['build_source'], lazy=True)
commit = gl_project.commits.get(commit_sha)
if not commit:
return None
return commit
@_catch_timeouts
@_catch_timeouts_and_errors
def lookup_user(self, email):
gl_client = self._get_authorized_client()
try:
result = gl_client.getusers(search=email)
if result is False:
result = gl_client.users.list(search=email)
if not result:
return None
[user] = result
return {
'username': user['username'],
'html_url': gl_client.host + '/' + user['username'],
'avatar_url': user['avatar_url']
'username': user.attributes['username'],
'html_url': user.attributes['web_url'],
'avatar_url': user.attributes['avatar_url']
}
except ValueError:
return None
@_catch_timeouts
@_catch_timeouts_and_errors
def get_metadata_for_commit(self, commit_sha, ref, repo):
gl_client = self._get_authorized_client()
commit = gl_client.getrepositorycommit(repo['id'], commit_sha)
commit = self.lookup_commit(repo.get_id(), commit_sha)
if commit is None:
return None
metadata = {
'commit': commit['id'],
'commit': commit.attributes['id'],
'ref': ref,
'default_branch': repo['default_branch'],
'git_url': repo['ssh_url_to_repo'],
'default_branch': repo.attributes['default_branch'],
'git_url': repo.attributes['ssh_url_to_repo'],
'commit_info': {
'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'],
'message': commit['message'],
'date': commit['committed_date'],
'url': os.path.join(repo.attributes['web_url'], 'commit', commit.attributes['id']),
'message': commit.attributes['message'],
'date': commit.attributes['committed_date'],
},
}
committer = None
if 'committer_email' in commit:
committer = self.lookup_user(commit['committer_email'])
if 'committer_email' in commit.attributes:
committer = self.lookup_user(commit.attributes['committer_email'])
author = None
if 'author_email' in commit:
author = self.lookup_user(commit['author_email'])
if 'author_email' in commit.attributes:
author = self.lookup_user(commit.attributes['author_email'])
if committer is not None:
metadata['commit_info']['committer'] = {
'username': committer['username'],
'avatar_url': committer['avatar_url'],
'url': gl_client.host + '/' + committer['username'],
'url': committer.get('http_url', ''),
}
if author is not None:
metadata['commit_info']['author'] = {
'username': author['username'],
'avatar_url': author['avatar_url'],
'url': gl_client.host + '/' + author['username']
'url': author.get('http_url', ''),
}
return metadata
@_catch_timeouts
@_catch_timeouts_and_errors
def manual_start(self, run_parameters=None):
gl_client = self._get_authorized_client()
repo = gl_client.getproject(self.config['build_source'])
if repo is False:
gl_project = gl_client.projects.get(self.config['build_source'])
if not gl_project:
raise TriggerStartException('Could not find repository')
def get_tag_sha(tag_name):
tags = gl_client.getrepositorytags(repo['id'])
if tags is False:
try:
tag = gl_project.tags.get(tag_name)
except gitlab.GitlabGetError:
raise TriggerStartException('Could not find tag in repository')
for tag in tags:
if tag['name'] == tag_name:
return tag['commit']['id']
raise TriggerStartException('Could not find tag in repository')
return tag.attributes['commit']['id']
def get_branch_sha(branch_name):
branch = gl_client.getbranch(repo['id'], branch_name)
if branch is False:
try:
branch = gl_project.branches.get(branch_name)
except gitlab.GitlabGetError:
raise TriggerStartException('Could not find branch in repository')
return branch['commit']['id']
return branch.attributes['commit']['id']
# Find the branch or tag to build.
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
repo['default_branch'])
gl_project.attributes['default_branch'])
metadata = self.get_metadata_for_commit(commit_sha, ref, repo)
metadata = self.get_metadata_for_commit(commit_sha, ref, gl_project)
return self.prepare_build(metadata, is_manual=True)
@_catch_timeouts
@_catch_timeouts_and_errors
def handle_trigger_request(self, request):
payload = request.get_json()
if not payload:
@ -552,12 +573,12 @@ class GitLabBuildTrigger(BuildTriggerHandler):
# Lookup the default branch.
gl_client = self._get_authorized_client()
repo = gl_client.getproject(self.config['build_source'])
if repo is False:
gl_project = gl_client.projects.get(self.config['build_source'])
if not gl_project:
logger.debug('Skipping GitLab build; project %s not found', self.config['build_source'])
raise InvalidPayloadException()
default_branch = repo['default_branch']
default_branch = gl_project.attributes['default_branch']
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
lookup_user=self.lookup_user,
lookup_commit=self.lookup_commit)