import logging import os from calendar import timegm from functools import wraps import dateutil.parser from app import app, gitlab_trigger from jsonschema import validate from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, TriggerDeactivationException, TriggerStartException, SkipRequestException, InvalidPayloadException, determine_build_ref, raise_if_skipped_build, find_matching_branches) from buildtrigger.basehandler import BuildTriggerHandler from util.security.ssh import generate_ssh_keypair from util.dict_wrappers import JSONPathDict, SafeDictSetter from endpoints.exception import ExternalServiceError import gitlab import requests logger = logging.getLogger(__name__) GITLAB_WEBHOOK_PAYLOAD_SCHEMA = { 'type': 'object', 'properties': { 'ref': { 'type': 'string', }, 'checkout_sha': { 'type': ['string', 'null'], }, 'repository': { 'type': 'object', 'properties': { 'git_ssh_url': { 'type': 'string', }, }, 'required': ['git_ssh_url'], }, 'commits': { 'type': 'array', 'items': { 'type': 'object', 'properties': { 'id': { 'type': 'string', }, 'url': { 'type': 'string', }, 'message': { 'type': 'string', }, 'timestamp': { 'type': 'string', }, 'author': { 'type': 'object', 'properties': { 'email': { 'type': 'string', }, }, 'required': ['email'], }, }, 'required': ['id', 'url', 'message', 'timestamp'], }, }, }, 'required': ['ref', 'checkout_sha', 'repository'], } _ACCESS_LEVEL_MAP = { 50: ("owner", True), 40: ("master", True), 30: ("developer", False), 20: ("reporter", False), 10: ("guest", False), } _PER_PAGE_COUNT = 20 def _catch_timeouts(func): @wraps(func) def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except requests.exceptions.Timeout: msg = 'Request to the GitLab API timed out' logger.exception(msg) raise ExternalServiceError(msg) return wrapper def _paginated_iterator(func, exc): """ Returns an iterator over invocations of the given function, automatically handling pagination. """ page = 0 while True: result = func(page=page, per_page=_PER_PAGE_COUNT) if result is False: raise exc counter = 0 for item in result: yield item counter = counter + 1 if counter < _PER_PAGE_COUNT: break page = page + 1 def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None, lookup_commit=None): """ Returns the Gitlab webhook JSON payload transformed into our own payload format. If the gl_payload is not valid, returns None. """ try: validate(gl_payload, GITLAB_WEBHOOK_PAYLOAD_SCHEMA) except Exception as exc: raise InvalidPayloadException(exc.message) payload = JSONPathDict(gl_payload) if payload['object_kind'] != 'push' and payload['object_kind'] != 'tag_push': # Unknown kind of webhook. raise SkipRequestException # Check for empty commits. The commits list will be empty if the branch is deleted. commits = payload['commits'] if payload['object_kind'] == 'push' and not commits: raise SkipRequestException config = SafeDictSetter() config['commit'] = payload['checkout_sha'] config['ref'] = payload['ref'] config['default_branch'] = default_branch config['git_url'] = payload['repository.git_ssh_url'] found_commit = JSONPathDict({}) if payload['object_kind'] == 'push': # Find the commit associated with the checkout_sha. Gitlab doesn't (necessary) send this in # any order, so we cannot simply index into the commits list. found_commit = None for commit in commits: if commit['id'] == payload['checkout_sha']: found_commit = JSONPathDict(commit) break if found_commit is None: raise SkipRequestException elif payload['object_kind'] == 'tag_push': # Gitlab doesn't send commit information for tag pushes (WHY?!), so we need to lookup the # commit SHA directly. if lookup_commit: found_commit_info = lookup_commit(payload['project_id'], payload['checkout_sha']) found_commit = JSONPathDict(found_commit_info or {}) config['commit_info.url'] = found_commit['url'] config['commit_info.message'] = found_commit['message'] config['commit_info.date'] = found_commit['timestamp'] # Note: Gitlab does not send full user information with the payload, so we have to # (optionally) look it up. author_email = found_commit['author.email'] or found_commit['author_email'] if lookup_user and author_email: author_info = lookup_user(author_email) if author_info: config['commit_info.author.username'] = author_info['username'] config['commit_info.author.url'] = author_info['html_url'] config['commit_info.author.avatar_url'] = author_info['avatar_url'] return config.dict_value() class GitLabBuildTrigger(BuildTriggerHandler): """ BuildTrigger for GitLab. """ @classmethod def service_name(cls): return 'gitlab' def _get_authorized_client(self): auth_token = self.auth_token or 'invalid' return gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=5) def is_active(self): return 'hook_id' in self.config @_catch_timeouts def activate(self, standard_webhook_url): config = self.config new_build_source = config['build_source'] gl_client = self._get_authorized_client() # Find the GitLab repository. repository = gl_client.getproject(new_build_source) if repository is False: msg = 'Unable to find GitLab repository for source: %s' % new_build_source raise TriggerActivationException(msg) # Add a deploy key to the repository. public_key, private_key = generate_ssh_keypair() config['credentials'] = [ { 'name': 'SSH Public Key', 'value': public_key, }, ] key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'], public_key) if key is False: msg = 'Unable to add deploy key to repository: %s' % new_build_source raise TriggerActivationException(msg) config['key_id'] = key['id'] # Add the webhook to the GitLab repository. hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True) if hook is False: msg = 'Unable to create webhook on repository: %s' % new_build_source raise TriggerActivationException(msg) config['hook_id'] = hook['id'] self.config = config return config, {'private_key': private_key} def deactivate(self): config = self.config gl_client = self._get_authorized_client() # Find the GitLab repository. repository = gl_client.getproject(config['build_source']) if repository is False: msg = 'Unable to find GitLab repository for source: %s' % config['build_source'] raise TriggerDeactivationException(msg) # Remove the webhook. success = gl_client.deleteprojecthook(repository['id'], config['hook_id']) if success is False: msg = 'Unable to remove hook: %s' % config['hook_id'] raise TriggerDeactivationException(msg) config.pop('hook_id', None) # Remove the key success = gl_client.deletedeploykey(repository['id'], config['key_id']) if success is False: msg = 'Unable to remove deploy key: %s' % config['key_id'] raise TriggerDeactivationException(msg) config.pop('key_id', None) self.config = config return config @_catch_timeouts def list_build_source_namespaces(self): gl_client = self._get_authorized_client() current_user = gl_client.currentuser() if current_user is False: raise RepositoryReadException('Unable to get current user') namespaces = {} repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException) for repo in repositories: namespace = repo.get('namespace') or {} if not namespace: continue namespace_id = namespace['id'] avatar_url = '' if 'avatar' in namespace: avatar_data = namespace.get('avatar') or {} avatar_url = avatar_data.get('url') elif 'owner' in repo: owner_data = repo.get('owner') or {} avatar_url = owner_data.get('avatar_url') if namespace_id in namespaces: namespaces[namespace_id]['score'] = namespaces[namespace_id]['score'] + 1 else: owner = namespace['name'] namespaces[namespace_id] = { 'personal': owner == current_user['username'], 'id': namespace['path'], 'title': namespace['name'], 'avatar_url': avatar_url, 'score': 1, 'url': gl_client.host + '/' + namespace['path'], } return BuildTriggerHandler.build_namespaces_response(namespaces) @_catch_timeouts def list_build_sources_for_namespace(self, namespace): def repo_view(repo): # Because *anything* can be None in GitLab API! permissions = repo.get('permissions') or {} group_access = permissions.get('group_access') or {} project_access = permissions.get('project_access') or {} missing_group_access = permissions.get('group_access') is None missing_project_access = permissions.get('project_access') is None access_level = max(group_access.get('access_level') or 0, project_access.get('access_level') or 0) has_admin_permission = _ACCESS_LEVEL_MAP.get(access_level, ("", False))[1] if missing_group_access or missing_project_access: # Default to has permission if we cannot check the permissions. This will allow our users # to select the repository and then GitLab's own checks will ensure that the webhook is # added only if allowed. # TODO: Do we want to display this differently in the UI? has_admin_permission = True view = { 'name': repo['path'], 'full_name': repo['path_with_namespace'], 'description': repo.get('description') or '', 'url': repo.get('web_url'), 'has_admin_permissions': has_admin_permission, 'private': repo.get('public', False) is False, } if repo.get('last_activity_at'): try: last_modified = dateutil.parser.parse(repo['last_activity_at']) view['last_updated'] = timegm(last_modified.utctimetuple()) except ValueError: logger.exception('Gitlab gave us an invalid last_activity_at: %s', last_modified) return view gl_client = self._get_authorized_client() repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException) repos = [repo_view(repo) for repo in repositories if repo['namespace']['path'] == namespace] return BuildTriggerHandler.build_sources_response(repos) @_catch_timeouts def list_build_subdirs(self): config = self.config gl_client = self._get_authorized_client() new_build_source = config['build_source'] repository = gl_client.getproject(new_build_source) if repository is False: msg = 'Unable to find GitLab repository for source: %s' % new_build_source raise RepositoryReadException(msg) repo_branches = gl_client.getbranches(repository['id']) if repo_branches is False: msg = 'Unable to find GitLab branches for source: %s' % new_build_source raise RepositoryReadException(msg) branches = [branch['name'] for branch in repo_branches] branches = find_matching_branches(config, branches) branches = branches or [repository['default_branch'] or 'master'] repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0]) if repo_tree is False: msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source raise RepositoryReadException(msg) return ["/" + node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])] @_catch_timeouts def load_dockerfile_contents(self): gl_client = self._get_authorized_client() path = self.get_dockerfile_path() repository = gl_client.getproject(self.config['build_source']) if repository is False: return None branches = self.list_field_values('branch_name') branches = find_matching_branches(self.config, branches) if branches == []: return None branch_name = branches[0] if repository['default_branch'] in branches: branch_name = repository['default_branch'] contents = gl_client.getrawfile(repository['id'], branch_name, path) if contents is False: return None return contents @_catch_timeouts def list_field_values(self, field_name, limit=None): if field_name == 'refs': branches = self.list_field_values('branch_name') tags = self.list_field_values('tag_name') return ([{'kind': 'branch', 'name': b} for b in branches] + [{'kind': 'tag', 'name': t} for t in tags]) gl_client = self._get_authorized_client() repo = gl_client.getproject(self.config['build_source']) if repo is False: return [] if field_name == 'tag_name': tags = gl_client.getrepositorytags(repo['id']) if tags is False: return [] if limit: tags = tags[0:limit] return [tag['name'] for tag in tags] if field_name == 'branch_name': branches = gl_client.getbranches(repo['id']) if branches is False: return [] if limit: branches = branches[0:limit] return [branch['name'] for branch in branches] return None def get_repository_url(self): return gitlab_trigger.get_public_url(self.config['build_source']) @_catch_timeouts def lookup_commit(self, repo_id, commit_sha): if repo_id is None: return None gl_client = self._get_authorized_client() commit = gl_client.getrepositorycommit(repo_id, commit_sha) if commit is False: return None return commit @_catch_timeouts def lookup_user(self, email): gl_client = self._get_authorized_client() try: result = gl_client.getusers(search=email) if result is False: return None [user] = result return { 'username': user['username'], 'html_url': gl_client.host + '/' + user['username'], 'avatar_url': user['avatar_url'] } except ValueError: return None @_catch_timeouts def get_metadata_for_commit(self, commit_sha, ref, repo): gl_client = self._get_authorized_client() commit = gl_client.getrepositorycommit(repo['id'], commit_sha) metadata = { 'commit': commit['id'], 'ref': ref, 'default_branch': repo['default_branch'], 'git_url': repo['ssh_url_to_repo'], 'commit_info': { 'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'], 'message': commit['message'], 'date': commit['committed_date'], }, } committer = None if 'committer_email' in commit: committer = self.lookup_user(commit['committer_email']) author = None if 'author_email' in commit: author = self.lookup_user(commit['author_email']) if committer is not None: metadata['commit_info']['committer'] = { 'username': committer['username'], 'avatar_url': committer['avatar_url'], 'url': gl_client.host + '/' + committer['username'], } if author is not None: metadata['commit_info']['author'] = { 'username': author['username'], 'avatar_url': author['avatar_url'], 'url': gl_client.host + '/' + author['username'] } return metadata @_catch_timeouts def manual_start(self, run_parameters=None): gl_client = self._get_authorized_client() repo = gl_client.getproject(self.config['build_source']) if repo is False: raise TriggerStartException('Could not find repository') def get_tag_sha(tag_name): tags = gl_client.getrepositorytags(repo['id']) if tags is False: raise TriggerStartException('Could not find tag in repository') for tag in tags: if tag['name'] == tag_name: return tag['commit']['id'] raise TriggerStartException('Could not find tag in repository') def get_branch_sha(branch_name): branch = gl_client.getbranch(repo['id'], branch_name) if branch is False: raise TriggerStartException('Could not find branch in repository') return branch['commit']['id'] # Find the branch or tag to build. (commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, repo['default_branch']) metadata = self.get_metadata_for_commit(commit_sha, ref, repo) return self.prepare_build(metadata, is_manual=True) @_catch_timeouts def handle_trigger_request(self, request): payload = request.get_json() if not payload: raise InvalidPayloadException() logger.debug('GitLab trigger payload %s', payload) # Lookup the default branch. gl_client = self._get_authorized_client() repo = gl_client.getproject(self.config['build_source']) if repo is False: logger.debug('Skipping GitLab build; project %s not found', self.config['build_source']) raise InvalidPayloadException() default_branch = repo['default_branch'] metadata = get_transformed_webhook_payload(payload, default_branch=default_branch, lookup_user=self.lookup_user, lookup_commit=self.lookup_commit) prepared = self.prepare_build(metadata) # Check if we should skip this build. raise_if_skipped_build(prepared, self.config) return prepared