import logging from app import app from jsonschema import validate from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException, TriggerDeactivationException, TriggerStartException, EmptyRepositoryException, ValidationRequestException, SkipRequestException, InvalidPayloadException, determine_build_ref, raise_if_skipped_build, find_matching_branches) from buildtrigger.basehandler import BuildTriggerHandler from util.security.ssh import generate_ssh_keypair from util.dict_wrappers import JSONPathDict, SafeDictSetter import gitlab logger = logging.getLogger(__name__) GITLAB_WEBHOOK_PAYLOAD_SCHEMA = { 'type': 'object', 'properties': { 'ref': { 'type': 'string', }, 'checkout_sha': { 'type': 'string', }, 'repository': { 'type': 'object', 'properties': { 'git_ssh_url': { 'type': 'string', }, }, 'required': ['git_ssh_url'], }, 'commits': { 'type': 'array', 'items': { 'type': 'object', 'properties': { 'url': { 'type': 'string', }, 'message': { 'type': 'string', }, 'timestamp': { 'type': 'string', }, 'author': { 'type': 'object', 'properties': { 'email': { 'type': 'string', }, }, 'required': ['email'], }, }, 'required': ['url', 'message', 'timestamp'], }, 'minItems': 1, } }, 'required': ['ref', 'checkout_sha', 'repository'], } def get_transformed_webhook_payload(gl_payload, default_branch=None, lookup_user=None): """ Returns the Gitlab webhook JSON payload transformed into our own payload format. If the gl_payload is not valid, returns None. """ try: validate(gl_payload, GITLAB_WEBHOOK_PAYLOAD_SCHEMA) except Exception as exc: raise InvalidPayloadException(exc.message) payload = JSONPathDict(gl_payload) config = SafeDictSetter() config['commit'] = payload['checkout_sha'] config['ref'] = payload['ref'] config['default_branch'] = default_branch config['git_url'] = payload['repository.git_ssh_url'] config['commit_info.url'] = payload['commits[0].url'] config['commit_info.message'] = payload['commits[0].message'] config['commit_info.date'] = payload['commits[0].timestamp'] # Note: Gitlab does not send full user information with the payload, so we have to # (optionally) look it up. author_email = payload['commits[0].author.email'] if lookup_user and author_email: author_info = lookup_user(author_email) if author_info: config['commit_info.author.username'] = author_info['username'] config['commit_info.author.url'] = author_info['html_url'] config['commit_info.author.avatar_url'] = author_info['avatar_url'] return config.dict_value() class GitLabBuildTrigger(BuildTriggerHandler): """ BuildTrigger for GitLab. """ @classmethod def service_name(cls): return 'gitlab' def _get_authorized_client(self): host = app.config.get('GITLAB_TRIGGER_CONFIG', {}).get('GITLAB_ENDPOINT', '') auth_token = self.auth_token or 'invalid' return gitlab.Gitlab(host, oauth_token=auth_token) def is_active(self): return 'hook_id' in self.config def activate(self, standard_webhook_url): config = self.config new_build_source = config['build_source'] gl_client = self._get_authorized_client() # Find the GitLab repository. repository = gl_client.getproject(new_build_source) if repository is False: msg = 'Unable to find GitLab repository for source: %s' % new_build_source raise TriggerActivationException(msg) # Add a deploy key to the repository. public_key, private_key = generate_ssh_keypair() config['credentials'] = [ { 'name': 'SSH Public Key', 'value': public_key, }, ] key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'], public_key) if key is False: msg = 'Unable to add deploy key to repository: %s' % new_build_source raise TriggerActivationException(msg) config['key_id'] = key['id'] # Add the webhook to the GitLab repository. hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True) if hook is False: msg = 'Unable to create webhook on repository: %s' % new_build_source raise TriggerActivationException(msg) config['hook_id'] = hook['id'] self.config = config return config, {'private_key': private_key} def deactivate(self): config = self.config gl_client = self._get_authorized_client() # Find the GitLab repository. repository = gl_client.getproject(config['build_source']) if repository is False: msg = 'Unable to find GitLab repository for source: %s' % config['build_source'] raise TriggerDeactivationException(msg) # Remove the webhook. success = gl_client.deleteprojecthook(repository['id'], config['hook_id']) if success is False: msg = 'Unable to remove hook: %s' % config['hook_id'] raise TriggerDeactivationException(msg) config.pop('hook_id', None) # Remove the key success = gl_client.deletedeploykey(repository['id'], config['key_id']) if success is False: msg = 'Unable to remove deploy key: %s' % config['key_id'] raise TriggerDeactivationException(msg) config.pop('key_id', None) self.config = config return config def list_build_sources(self): gl_client = self._get_authorized_client() current_user = gl_client.currentuser() if current_user is False: raise RepositoryReadException('Unable to get current user') repositories = gl_client.getprojects() if repositories is False: raise RepositoryReadException('Unable to list user repositories') namespaces = {} for repo in repositories: owner = repo['namespace']['name'] if not owner in namespaces: namespaces[owner] = { 'personal': owner == current_user['username'], 'repos': [], 'info': { 'name': owner, } } namespaces[owner]['repos'].append(repo['path_with_namespace']) return namespaces.values() def list_build_subdirs(self): config = self.config gl_client = self._get_authorized_client() new_build_source = config['build_source'] repository = gl_client.getproject(new_build_source) if repository is False: msg = 'Unable to find GitLab repository for source: %s' % new_build_source raise RepositoryReadException(msg) repo_branches = gl_client.getbranches(repository['id']) if repo_branches is False: msg = 'Unable to find GitLab branches for source: %s' % new_build_source raise RepositoryReadException(msg) branches = [branch['name'] for branch in repo_branches] branches = find_matching_branches(config, branches) branches = branches or [repository['default_branch'] or 'master'] repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0]) if repo_tree is False: msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source raise RepositoryReadException(msg) for node in repo_tree: if node['name'] == 'Dockerfile': return ['/'] return [] def load_dockerfile_contents(self): gl_client = self._get_authorized_client() path = self.get_dockerfile_path() repository = gl_client.getproject(self.config['build_source']) if repository is False: return None branches = self.list_field_values('branch_name') branches = find_matching_branches(self.config, branches) if branches == []: return None branch_name = branches[0] if repository['default_branch'] in branches: branch_name = repository['default_branch'] contents = gl_client.getrawfile(repository['id'], branch_name, path) if contents is False: return None return contents def list_field_values(self, field_name, limit=None): if field_name == 'refs': branches = self.list_field_values('branch_name') tags = self.list_field_values('tag_name') return ([{'kind': 'branch', 'name': b} for b in branches] + [{'kind': 'tag', 'name': t} for t in tags]) gl_client = self._get_authorized_client() repo = gl_client.getproject(self.config['build_source']) if repo is False: return [] if field_name == 'tag_name': tags = gl_client.getrepositorytags(repo['id']) if tags is False: return [] if limit: tags = tags[0:limit] return [tag['name'] for tag in tags] if field_name == 'branch_name': branches = gl_client.getbranches(repo['id']) if branches is False: return [] if limit: branches = branches[0:limit] return [branch['name'] for branch in branches] return None def get_repository_url(self): gl_client = self._get_authorized_client() repository = gl_client.getproject(self.config['build_source']) if repository is False: return None return '%s/%s' % (gl_client.host, repository['path_with_namespace']) def lookup_user(self, email): gl_client = self._get_authorized_client() try: [user] = gl_client.getusers(search=email) return { 'username': user['username'], 'html_url': gl_client.host + '/' + user['username'], 'avatar_url': user['avatar_url'] } except ValueError: return None def get_metadata_for_commit(self, commit_sha, ref, repo): gl_client = self._get_authorized_client() commit = gl_client.getrepositorycommit(repo['id'], commit_sha) metadata = { 'commit': commit['id'], 'ref': ref, 'default_branch': repo['default_branch'], 'git_url': repo['ssh_url_to_repo'], 'commit_info': { 'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'], 'message': commit['message'], 'date': commit['committed_date'], }, } committer = None if 'committer_email' in commit: committer = self.lookup_user(commit['committer_email']) author = None if 'author_email' in commit: author = self.lookup_user(commit['author_email']) if committer is not None: metadata['commit_info']['committer'] = { 'username': committer['username'], 'avatar_url': committer['avatar_url'], 'url': gl_client.host + '/' + committer['username'], } if author is not None: metadata['commit_info']['author'] = { 'username': author['username'], 'avatar_url': author['avatar_url'], 'url': gl_client.host + '/' + author['username'] } return metadata def manual_start(self, run_parameters=None): gl_client = self._get_authorized_client() repo = gl_client.getproject(self.config['build_source']) if repo is False: raise TriggerStartException('Could not find repository') def get_tag_sha(tag_name): tags = gl_client.getrepositorytags(repo['id']) if tags is False: raise TriggerStartException('Could not find tags') for tag in tags: if tag['name'] == tag_name: return tag['commit']['id'] raise TriggerStartException('Could not find commit') def get_branch_sha(branch_name): branch = gl_client.getbranch(repo['id'], branch_name) if branch is False: raise TriggerStartException('Could not find branch') return branch['commit']['id'] # Find the branch or tag to build. (commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha, repo['default_branch']) metadata = self.get_metadata_for_commit(commit_sha, ref, repo) return self.prepare_build(metadata, is_manual=True) def handle_trigger_request(self, request): payload = request.get_json() if not payload: raise SkipRequestException() # Lookup the default branch. default_branch = None gl_client = self._get_authorized_client() repo = gl_client.getproject(self.config['build_source']) if repo is not False: default_branch = repo['default_branch'] lookup_user = self.lookup_user logger.debug('GitLab trigger payload %s', payload) metadata = get_transformed_webhook_payload(payload, default_branch=default_branch, lookup_user=lookup_user) prepared = self.prepare_build(metadata) # Check if we should skip this build. raise_if_skipped_build(prepared, self.config) return prepared