Merge pull request #3110 from quay/joseph.schorr/QUAY-966/gitlab-v4
Reimplement GitLab trigger handler using the V4 API library
This commit is contained in:
Joseph Schorr
GitHub
commit
1be22a9a56
9 changed files with 912 additions and 393 deletions
|
|
@ -1,28 +1,26 @@
|
|||
import os.path
|
||||
import logging
|
||||
import os
|
||||
|
||||
from calendar import timegm
|
||||
from functools import wraps
|
||||
|
||||
import dateutil.parser
|
||||
|
||||
from app import app, gitlab_trigger
|
||||
import gitlab
|
||||
import requests
|
||||
|
||||
from jsonschema import validate
|
||||
|
||||
from app import app, gitlab_trigger
|
||||
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
|
||||
TriggerDeactivationException, TriggerStartException,
|
||||
SkipRequestException, InvalidPayloadException,
|
||||
TriggerAuthException,
|
||||
determine_build_ref, raise_if_skipped_build,
|
||||
find_matching_branches)
|
||||
|
||||
from buildtrigger.basehandler import BuildTriggerHandler
|
||||
|
||||
from endpoints.exception import ExternalServiceError
|
||||
from util.security.ssh import generate_ssh_keypair
|
||||
from util.dict_wrappers import JSONPathDict, SafeDictSetter
|
||||
from endpoints.exception import ExternalServiceError
|
||||
|
||||
import gitlab
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -89,7 +87,7 @@ _ACCESS_LEVEL_MAP = {
|
|||
_PER_PAGE_COUNT = 20
|
||||
|
||||
|
||||
def _catch_timeouts(func):
|
||||
def _catch_timeouts_and_errors(func):
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
|
|
@ -98,17 +96,21 @@ def _catch_timeouts(func):
|
|||
msg = 'Request to the GitLab API timed out'
|
||||
logger.exception(msg)
|
||||
raise ExternalServiceError(msg)
|
||||
except gitlab.GitlabError:
|
||||
msg = 'GitLab API error. Please contact support.'
|
||||
logger.exception(msg)
|
||||
raise ExternalServiceError(msg)
|
||||
return wrapper
|
||||
|
||||
|
||||
def _paginated_iterator(func, exc):
|
||||
def _paginated_iterator(func, exc, **kwargs):
|
||||
""" Returns an iterator over invocations of the given function, automatically handling
|
||||
pagination.
|
||||
"""
|
||||
page = 0
|
||||
page = 1
|
||||
while True:
|
||||
result = func(page=page, per_page=_PER_PAGE_COUNT)
|
||||
if result is False:
|
||||
result = func(page=page, per_page=_PER_PAGE_COUNT, **kwargs)
|
||||
if result is None or result is False:
|
||||
raise exc
|
||||
|
||||
counter = 0
|
||||
|
|
@ -196,20 +198,28 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
def _get_authorized_client(self):
|
||||
auth_token = self.auth_token or 'invalid'
|
||||
return gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=5)
|
||||
api_version = self.config.get('API_VERSION', '4')
|
||||
client = gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=20,
|
||||
api_version=api_version)
|
||||
try:
|
||||
client.auth()
|
||||
except gitlab.GitlabGetError as ex:
|
||||
raise TriggerAuthException(ex.message)
|
||||
|
||||
return client
|
||||
|
||||
def is_active(self):
|
||||
return 'hook_id' in self.config
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def activate(self, standard_webhook_url):
|
||||
config = self.config
|
||||
new_build_source = config['build_source']
|
||||
gl_client = self._get_authorized_client()
|
||||
|
||||
# Find the GitLab repository.
|
||||
repository = gl_client.getproject(new_build_source)
|
||||
if repository is False:
|
||||
gl_project = gl_client.projects.get(new_build_source)
|
||||
if not gl_project:
|
||||
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
|
|
@ -221,20 +231,28 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
'value': public_key,
|
||||
},
|
||||
]
|
||||
key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'],
|
||||
public_key)
|
||||
if key is False:
|
||||
|
||||
key = gl_project.keys.create({
|
||||
'title': '%s Builder' % app.config['REGISTRY_TITLE'],
|
||||
'key': public_key,
|
||||
})
|
||||
|
||||
if not key:
|
||||
msg = 'Unable to add deploy key to repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
config['key_id'] = key['id']
|
||||
|
||||
config['key_id'] = key.get_id()
|
||||
|
||||
# Add the webhook to the GitLab repository.
|
||||
hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True)
|
||||
if hook is False:
|
||||
hook = gl_project.hooks.create({
|
||||
'url': standard_webhook_url,
|
||||
'push': True,
|
||||
})
|
||||
if not hook:
|
||||
msg = 'Unable to create webhook on repository: %s' % new_build_source
|
||||
raise TriggerActivationException(msg)
|
||||
|
||||
config['hook_id'] = hook['id']
|
||||
config['hook_id'] = hook.get_id()
|
||||
self.config = config
|
||||
return config, {'private_key': private_key}
|
||||
|
||||
|
|
@ -243,72 +261,71 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
gl_client = self._get_authorized_client()
|
||||
|
||||
# Find the GitLab repository.
|
||||
repository = gl_client.getproject(config['build_source'])
|
||||
if repository is False:
|
||||
gl_project = gl_client.projects.get(config['build_source'])
|
||||
if not gl_project:
|
||||
msg = 'Unable to find GitLab repository for source: %s' % config['build_source']
|
||||
raise TriggerDeactivationException(msg)
|
||||
|
||||
# Remove the webhook.
|
||||
success = gl_client.deleteprojecthook(repository['id'], config['hook_id'])
|
||||
if success is False:
|
||||
msg = 'Unable to remove hook: %s' % config['hook_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
gl_project.hooks.delete(config['hook_id'])
|
||||
config.pop('hook_id', None)
|
||||
|
||||
# Remove the key
|
||||
success = gl_client.deletedeploykey(repository['id'], config['key_id'])
|
||||
if success is False:
|
||||
msg = 'Unable to remove deploy key: %s' % config['key_id']
|
||||
raise TriggerDeactivationException(msg)
|
||||
gl_project.keys.delete(config['key_id'])
|
||||
config.pop('key_id', None)
|
||||
|
||||
self.config = config
|
||||
return config
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def list_build_source_namespaces(self):
|
||||
gl_client = self._get_authorized_client()
|
||||
current_user = gl_client.currentuser()
|
||||
if current_user is False:
|
||||
current_user = gl_client.user
|
||||
if not current_user:
|
||||
raise RepositoryReadException('Unable to get current user')
|
||||
|
||||
namespaces = {}
|
||||
repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
|
||||
for repo in repositories:
|
||||
namespace = repo.get('namespace') or {}
|
||||
if not namespace:
|
||||
for namespace in _paginated_iterator(gl_client.namespaces.list, RepositoryReadException):
|
||||
namespace_id = namespace.get_id()
|
||||
|
||||
# Retrieve the namespace as a user or group.
|
||||
namespace_obj = self._get_namespace(gl_client, namespace)
|
||||
if namespace_obj is None:
|
||||
logger.warning('Could not load details for namespace %s', namespace_id)
|
||||
continue
|
||||
|
||||
namespace_id = namespace['id']
|
||||
|
||||
avatar_url = ''
|
||||
if 'avatar' in namespace:
|
||||
avatar_data = namespace.get('avatar') or {}
|
||||
avatar_url = avatar_data.get('url')
|
||||
elif 'owner' in repo:
|
||||
owner_data = repo.get('owner') or {}
|
||||
avatar_url = owner_data.get('avatar_url')
|
||||
|
||||
if namespace_id in namespaces:
|
||||
namespaces[namespace_id]['score'] = namespaces[namespace_id]['score'] + 1
|
||||
else:
|
||||
owner = namespace['name']
|
||||
owner = namespace.attributes['name']
|
||||
namespaces[namespace_id] = {
|
||||
'personal': owner == current_user['username'],
|
||||
'id': namespace['path'],
|
||||
'title': namespace['name'],
|
||||
'avatar_url': avatar_url,
|
||||
'personal': owner == current_user.attributes['username'],
|
||||
'id': str(namespace_id),
|
||||
'title': namespace.attributes['name'],
|
||||
'avatar_url': namespace_obj.attributes.get('avatar_url', ''),
|
||||
'score': 1,
|
||||
'url': gl_client.host + '/' + namespace['path'],
|
||||
'url': namespace_obj.attributes.get('web_url', ''),
|
||||
}
|
||||
|
||||
return BuildTriggerHandler.build_namespaces_response(namespaces)
|
||||
|
||||
@_catch_timeouts
|
||||
def list_build_sources_for_namespace(self, namespace):
|
||||
def _get_namespace(self, gl_client, gl_namespace, lazy=False):
|
||||
try:
|
||||
if gl_namespace.attributes['kind'] == 'group':
|
||||
return gl_client.groups.get(gl_namespace.attributes['name'], lazy=lazy)
|
||||
|
||||
return gl_client.users.get(gl_namespace.attributes['name'], lazy=lazy)
|
||||
except gitlab.GitlabGetError:
|
||||
return None
|
||||
|
||||
@_catch_timeouts_and_errors
|
||||
def list_build_sources_for_namespace(self, namespace_id):
|
||||
if not namespace_id:
|
||||
return []
|
||||
|
||||
def repo_view(repo):
|
||||
# Because *anything* can be None in GitLab API!
|
||||
permissions = repo.get('permissions') or {}
|
||||
permissions = repo.attributes.get('permissions') or {}
|
||||
group_access = permissions.get('group_access') or {}
|
||||
project_access = permissions.get('project_access') or {}
|
||||
|
||||
|
|
@ -327,17 +344,17 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
has_admin_permission = True
|
||||
|
||||
view = {
|
||||
'name': repo['path'],
|
||||
'full_name': repo['path_with_namespace'],
|
||||
'description': repo.get('description') or '',
|
||||
'url': repo.get('web_url'),
|
||||
'name': repo.attributes['path'],
|
||||
'full_name': repo.attributes['path_with_namespace'],
|
||||
'description': repo.attributes.get('description') or '',
|
||||
'url': repo.attributes.get('web_url'),
|
||||
'has_admin_permissions': has_admin_permission,
|
||||
'private': repo.get('public', False) is False,
|
||||
'private': repo.attributes.get('visibility') == 'private',
|
||||
}
|
||||
|
||||
if repo.get('last_activity_at'):
|
||||
if repo.attributes.get('last_activity_at'):
|
||||
try:
|
||||
last_modified = dateutil.parser.parse(repo['last_activity_at'])
|
||||
last_modified = dateutil.parser.parse(repo.attributes['last_activity_at'])
|
||||
view['last_updated'] = timegm(last_modified.utctimetuple())
|
||||
except ValueError:
|
||||
logger.exception('Gitlab gave us an invalid last_activity_at: %s', last_modified)
|
||||
|
|
@ -345,44 +362,50 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
return view
|
||||
|
||||
gl_client = self._get_authorized_client()
|
||||
repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
|
||||
repos = [repo_view(repo) for repo in repositories if repo['namespace']['path'] == namespace]
|
||||
return BuildTriggerHandler.build_sources_response(repos)
|
||||
|
||||
@_catch_timeouts
|
||||
try:
|
||||
gl_namespace = gl_client.namespaces.get(namespace_id)
|
||||
except gitlab.GitlabGetError:
|
||||
return []
|
||||
|
||||
namespace_obj = self._get_namespace(gl_client, gl_namespace, lazy=True)
|
||||
repositories = _paginated_iterator(namespace_obj.projects.list, RepositoryReadException)
|
||||
return BuildTriggerHandler.build_sources_response([repo_view(repo) for repo in repositories])
|
||||
|
||||
@_catch_timeouts_and_errors
|
||||
def list_build_subdirs(self):
|
||||
config = self.config
|
||||
gl_client = self._get_authorized_client()
|
||||
new_build_source = config['build_source']
|
||||
|
||||
repository = gl_client.getproject(new_build_source)
|
||||
if repository is False:
|
||||
gl_project = gl_client.projects.get(new_build_source)
|
||||
if not gl_project:
|
||||
msg = 'Unable to find GitLab repository for source: %s' % new_build_source
|
||||
raise RepositoryReadException(msg)
|
||||
|
||||
repo_branches = gl_client.getbranches(repository['id'])
|
||||
if repo_branches is False:
|
||||
repo_branches = gl_project.branches.list()
|
||||
if not repo_branches:
|
||||
msg = 'Unable to find GitLab branches for source: %s' % new_build_source
|
||||
raise RepositoryReadException(msg)
|
||||
|
||||
branches = [branch['name'] for branch in repo_branches]
|
||||
branches = [branch.attributes['name'] for branch in repo_branches]
|
||||
branches = find_matching_branches(config, branches)
|
||||
branches = branches or [repository['default_branch'] or 'master']
|
||||
branches = branches or [gl_project.attributes['default_branch'] or 'master']
|
||||
|
||||
repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0])
|
||||
if repo_tree is False:
|
||||
repo_tree = gl_project.repository_tree(ref=branches[0])
|
||||
if not repo_tree:
|
||||
msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
|
||||
raise RepositoryReadException(msg)
|
||||
|
||||
return ["/" + node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
|
||||
return [node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def load_dockerfile_contents(self):
|
||||
gl_client = self._get_authorized_client()
|
||||
path = self.get_dockerfile_path()
|
||||
|
||||
repository = gl_client.getproject(self.config['build_source'])
|
||||
if repository is False:
|
||||
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||
if not gl_project:
|
||||
return None
|
||||
|
||||
branches = self.list_field_values('branch_name')
|
||||
|
|
@ -391,16 +414,15 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
return None
|
||||
|
||||
branch_name = branches[0]
|
||||
if repository['default_branch'] in branches:
|
||||
branch_name = repository['default_branch']
|
||||
if gl_project.attributes['default_branch'] in branches:
|
||||
branch_name = gl_project.attributes['default_branch']
|
||||
|
||||
contents = gl_client.getrawfile(repository['id'], branch_name, path)
|
||||
if contents is False:
|
||||
try:
|
||||
return gl_project.files.get(path, branch_name).decode()
|
||||
except gitlab.GitlabGetError:
|
||||
return None
|
||||
|
||||
return contents
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def list_field_values(self, field_name, limit=None):
|
||||
if field_name == 'refs':
|
||||
branches = self.list_field_values('branch_name')
|
||||
|
|
@ -410,139 +432,138 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
[{'kind': 'tag', 'name': t} for t in tags])
|
||||
|
||||
gl_client = self._get_authorized_client()
|
||||
repo = gl_client.getproject(self.config['build_source'])
|
||||
if repo is False:
|
||||
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||
if not gl_project:
|
||||
return []
|
||||
|
||||
if field_name == 'tag_name':
|
||||
tags = gl_client.getrepositorytags(repo['id'])
|
||||
if tags is False:
|
||||
tags = gl_project.tags.list()
|
||||
if not tags:
|
||||
return []
|
||||
|
||||
if limit:
|
||||
tags = tags[0:limit]
|
||||
|
||||
return [tag['name'] for tag in tags]
|
||||
return [tag.attributes['name'] for tag in tags]
|
||||
|
||||
if field_name == 'branch_name':
|
||||
branches = gl_client.getbranches(repo['id'])
|
||||
if branches is False:
|
||||
branches = gl_project.branches.list()
|
||||
if not branches:
|
||||
return []
|
||||
|
||||
if limit:
|
||||
branches = branches[0:limit]
|
||||
|
||||
return [branch['name'] for branch in branches]
|
||||
return [branch.attributes['name'] for branch in branches]
|
||||
|
||||
return None
|
||||
|
||||
def get_repository_url(self):
|
||||
return gitlab_trigger.get_public_url(self.config['build_source'])
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def lookup_commit(self, repo_id, commit_sha):
|
||||
if repo_id is None:
|
||||
return None
|
||||
|
||||
gl_client = self._get_authorized_client()
|
||||
commit = gl_client.getrepositorycommit(repo_id, commit_sha)
|
||||
if commit is False:
|
||||
gl_project = gl_client.projects.get(self.config['build_source'], lazy=True)
|
||||
commit = gl_project.commits.get(commit_sha)
|
||||
if not commit:
|
||||
return None
|
||||
|
||||
return commit
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def lookup_user(self, email):
|
||||
gl_client = self._get_authorized_client()
|
||||
try:
|
||||
result = gl_client.getusers(search=email)
|
||||
if result is False:
|
||||
result = gl_client.users.list(search=email)
|
||||
if not result:
|
||||
return None
|
||||
|
||||
[user] = result
|
||||
return {
|
||||
'username': user['username'],
|
||||
'html_url': gl_client.host + '/' + user['username'],
|
||||
'avatar_url': user['avatar_url']
|
||||
'username': user.attributes['username'],
|
||||
'html_url': user.attributes['web_url'],
|
||||
'avatar_url': user.attributes['avatar_url']
|
||||
}
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def get_metadata_for_commit(self, commit_sha, ref, repo):
|
||||
gl_client = self._get_authorized_client()
|
||||
commit = gl_client.getrepositorycommit(repo['id'], commit_sha)
|
||||
commit = self.lookup_commit(repo.get_id(), commit_sha)
|
||||
if commit is None:
|
||||
return None
|
||||
|
||||
metadata = {
|
||||
'commit': commit['id'],
|
||||
'commit': commit.attributes['id'],
|
||||
'ref': ref,
|
||||
'default_branch': repo['default_branch'],
|
||||
'git_url': repo['ssh_url_to_repo'],
|
||||
'default_branch': repo.attributes['default_branch'],
|
||||
'git_url': repo.attributes['ssh_url_to_repo'],
|
||||
'commit_info': {
|
||||
'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'],
|
||||
'message': commit['message'],
|
||||
'date': commit['committed_date'],
|
||||
'url': os.path.join(repo.attributes['web_url'], 'commit', commit.attributes['id']),
|
||||
'message': commit.attributes['message'],
|
||||
'date': commit.attributes['committed_date'],
|
||||
},
|
||||
}
|
||||
|
||||
committer = None
|
||||
if 'committer_email' in commit:
|
||||
committer = self.lookup_user(commit['committer_email'])
|
||||
if 'committer_email' in commit.attributes:
|
||||
committer = self.lookup_user(commit.attributes['committer_email'])
|
||||
|
||||
author = None
|
||||
if 'author_email' in commit:
|
||||
author = self.lookup_user(commit['author_email'])
|
||||
if 'author_email' in commit.attributes:
|
||||
author = self.lookup_user(commit.attributes['author_email'])
|
||||
|
||||
if committer is not None:
|
||||
metadata['commit_info']['committer'] = {
|
||||
'username': committer['username'],
|
||||
'avatar_url': committer['avatar_url'],
|
||||
'url': gl_client.host + '/' + committer['username'],
|
||||
'url': committer.get('http_url', ''),
|
||||
}
|
||||
|
||||
if author is not None:
|
||||
metadata['commit_info']['author'] = {
|
||||
'username': author['username'],
|
||||
'avatar_url': author['avatar_url'],
|
||||
'url': gl_client.host + '/' + author['username']
|
||||
'url': author.get('http_url', ''),
|
||||
}
|
||||
|
||||
return metadata
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def manual_start(self, run_parameters=None):
|
||||
gl_client = self._get_authorized_client()
|
||||
|
||||
repo = gl_client.getproject(self.config['build_source'])
|
||||
if repo is False:
|
||||
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||
if not gl_project:
|
||||
raise TriggerStartException('Could not find repository')
|
||||
|
||||
def get_tag_sha(tag_name):
|
||||
tags = gl_client.getrepositorytags(repo['id'])
|
||||
if tags is False:
|
||||
try:
|
||||
tag = gl_project.tags.get(tag_name)
|
||||
except gitlab.GitlabGetError:
|
||||
raise TriggerStartException('Could not find tag in repository')
|
||||
|
||||
for tag in tags:
|
||||
if tag['name'] == tag_name:
|
||||
return tag['commit']['id']
|
||||
|
||||
raise TriggerStartException('Could not find tag in repository')
|
||||
return tag.attributes['commit']['id']
|
||||
|
||||
def get_branch_sha(branch_name):
|
||||
branch = gl_client.getbranch(repo['id'], branch_name)
|
||||
if branch is False:
|
||||
try:
|
||||
branch = gl_project.branches.get(branch_name)
|
||||
except gitlab.GitlabGetError:
|
||||
raise TriggerStartException('Could not find branch in repository')
|
||||
|
||||
return branch['commit']['id']
|
||||
return branch.attributes['commit']['id']
|
||||
|
||||
# Find the branch or tag to build.
|
||||
(commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
|
||||
repo['default_branch'])
|
||||
gl_project.attributes['default_branch'])
|
||||
|
||||
metadata = self.get_metadata_for_commit(commit_sha, ref, repo)
|
||||
metadata = self.get_metadata_for_commit(commit_sha, ref, gl_project)
|
||||
return self.prepare_build(metadata, is_manual=True)
|
||||
|
||||
@_catch_timeouts
|
||||
@_catch_timeouts_and_errors
|
||||
def handle_trigger_request(self, request):
|
||||
payload = request.get_json()
|
||||
if not payload:
|
||||
|
|
@ -552,12 +573,12 @@ class GitLabBuildTrigger(BuildTriggerHandler):
|
|||
|
||||
# Lookup the default branch.
|
||||
gl_client = self._get_authorized_client()
|
||||
repo = gl_client.getproject(self.config['build_source'])
|
||||
if repo is False:
|
||||
gl_project = gl_client.projects.get(self.config['build_source'])
|
||||
if not gl_project:
|
||||
logger.debug('Skipping GitLab build; project %s not found', self.config['build_source'])
|
||||
raise InvalidPayloadException()
|
||||
|
||||
default_branch = repo['default_branch']
|
||||
default_branch = gl_project.attributes['default_branch']
|
||||
metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
|
||||
lookup_user=self.lookup_user,
|
||||
lookup_commit=self.lookup_commit)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,588 @@
|
|||
from datetime import datetime
|
||||
from mock import Mock
|
||||
import base64
|
||||
import json
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import gitlab
|
||||
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from buildtrigger.gitlabhandler import GitLabBuildTrigger
|
||||
from util.morecollections import AttrDict
|
||||
|
||||
def get_gitlab_trigger(dockerfile_path=''):
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab')
|
||||
def catchall_handler(url, request):
|
||||
return {'status_code': 404}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/users$')
|
||||
def users_handler(url, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
if url.query.find('knownuser') < 0:
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([]),
|
||||
}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([
|
||||
{
|
||||
"id": 1,
|
||||
"username": "knownuser",
|
||||
"name": "Known User",
|
||||
"state": "active",
|
||||
"avatar_url": "avatarurl",
|
||||
"web_url": "https://bitbucket.org/knownuser",
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/user$')
|
||||
def user_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": 1,
|
||||
"username": "john_smith",
|
||||
"email": "john@example.com",
|
||||
"name": "John Smith",
|
||||
"state": "active",
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/foo%2Fbar$')
|
||||
def project_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": 4,
|
||||
"description": None,
|
||||
"default_branch": "master",
|
||||
"visibility": "private",
|
||||
"path_with_namespace": "someorg/somerepo",
|
||||
"ssh_url_to_repo": "git@example.com:someorg/somerepo.git",
|
||||
"web_url": "http://example.com/someorg/somerepo",
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tree$')
|
||||
def project_tree_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([
|
||||
{
|
||||
"id": "a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
|
||||
"name": "Dockerfile",
|
||||
"type": "tree",
|
||||
"path": "files/Dockerfile",
|
||||
"mode": "040000",
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tags$')
|
||||
def project_tags_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([
|
||||
{
|
||||
'name': 'sometag',
|
||||
'commit': {
|
||||
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'someothertag',
|
||||
'commit': {
|
||||
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||
},
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/branches$')
|
||||
def project_branches_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([
|
||||
{
|
||||
'name': 'master',
|
||||
'commit': {
|
||||
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'otherbranch',
|
||||
'commit': {
|
||||
'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
|
||||
},
|
||||
},
|
||||
]),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/branches/master$')
|
||||
def project_branch_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"name": "master",
|
||||
"merged": True,
|
||||
"protected": True,
|
||||
"developers_can_push": False,
|
||||
"developers_can_merge": False,
|
||||
"commit": {
|
||||
"author_email": "john@example.com",
|
||||
"author_name": "John Smith",
|
||||
"authored_date": "2012-06-27T05:51:39-07:00",
|
||||
"committed_date": "2012-06-28T03:44:20-07:00",
|
||||
"committer_email": "john@example.com",
|
||||
"committer_name": "John Smith",
|
||||
"id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||
"short_id": "7b5c3cc",
|
||||
"title": "add projects API",
|
||||
"message": "add projects API",
|
||||
"parent_ids": [
|
||||
"4ad91d3c1144c406e50c7b33bae684bd6837faf8",
|
||||
],
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces/someorg$')
|
||||
def namespace_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": 2,
|
||||
"name": "someorg",
|
||||
"path": "someorg",
|
||||
"kind": "group",
|
||||
"full_path": "someorg",
|
||||
"parent_id": None,
|
||||
"members_count_with_descendants": 2
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces/knownuser$')
|
||||
def user_namespace_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": 2,
|
||||
"name": "knownuser",
|
||||
"path": "knownuser",
|
||||
"kind": "user",
|
||||
"full_path": "knownuser",
|
||||
"parent_id": None,
|
||||
"members_count_with_descendants": 2
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces(/)?$')
|
||||
def namespaces_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([{
|
||||
"id": 2,
|
||||
"name": "someorg",
|
||||
"path": "someorg",
|
||||
"kind": "group",
|
||||
"full_path": "someorg",
|
||||
"parent_id": None,
|
||||
"members_count_with_descendants": 2
|
||||
}]),
|
||||
}
|
||||
|
||||
|
||||
def get_projects_handler(add_permissions_block):
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/groups/someorg/projects$')
|
||||
def projects_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
permissions_block = {
|
||||
"project_access": {
|
||||
"access_level": 10,
|
||||
"notification_level": 3
|
||||
},
|
||||
"group_access": {
|
||||
"access_level": 20,
|
||||
"notification_level": 3
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([{
|
||||
"id": 4,
|
||||
"name": "Some project",
|
||||
"description": None,
|
||||
"default_branch": "master",
|
||||
"visibility": "private",
|
||||
"path": "someproject",
|
||||
"path_with_namespace": "someorg/someproject",
|
||||
"last_activity_at": "2013-09-30T13:46:02Z",
|
||||
"web_url": "http://example.com/someorg/someproject",
|
||||
"permissions": permissions_block if add_permissions_block else None,
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"name": "Another project",
|
||||
"description": None,
|
||||
"default_branch": "master",
|
||||
"visibility": "public",
|
||||
"path": "anotherproject",
|
||||
"path_with_namespace": "someorg/anotherproject",
|
||||
"last_activity_at": "2013-09-30T13:46:02Z",
|
||||
"web_url": "http://example.com/someorg/anotherproject",
|
||||
}]),
|
||||
}
|
||||
return projects_handler
|
||||
|
||||
|
||||
def get_group_handler(null_avatar):
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/groups/someorg$')
|
||||
def group_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": 1,
|
||||
"name": "SomeOrg Group",
|
||||
"path": "someorg",
|
||||
"description": "An interesting group",
|
||||
"visibility": "public",
|
||||
"lfs_enabled": True,
|
||||
"avatar_url": 'avatar_url' if not null_avatar else None,
|
||||
"web_url": "http://gitlab.com/groups/someorg",
|
||||
"request_access_enabled": False,
|
||||
"full_name": "SomeOrg Group",
|
||||
"full_path": "someorg",
|
||||
"parent_id": None,
|
||||
}),
|
||||
}
|
||||
return group_handler
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/files/Dockerfile$')
|
||||
def dockerfile_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"file_name": "Dockerfile",
|
||||
"file_path": "Dockerfile",
|
||||
"size": 10,
|
||||
"encoding": "base64",
|
||||
"content": base64.b64encode('hello world'),
|
||||
"ref": "master",
|
||||
"blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
|
||||
"commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
|
||||
"last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d"
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/files/somesubdir%2FDockerfile$')
|
||||
def sub_dockerfile_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"file_name": "Dockerfile",
|
||||
"file_path": "somesubdir/Dockerfile",
|
||||
"size": 10,
|
||||
"encoding": "base64",
|
||||
"content": base64.b64encode('hi universe'),
|
||||
"ref": "master",
|
||||
"blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
|
||||
"commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
|
||||
"last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d"
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tags/sometag$')
|
||||
def tag_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"name": "sometag",
|
||||
"message": "some cool message",
|
||||
"target": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||
"commit": {
|
||||
"id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||
"short_id": "60a8ff03",
|
||||
"title": "Initial commit",
|
||||
"created_at": "2017-07-26T11:08:53.000+02:00",
|
||||
"parent_ids": [
|
||||
"f61c062ff8bcbdb00e0a1b3317a91aed6ceee06b"
|
||||
],
|
||||
"message": "v5.0.0\n",
|
||||
"author_name": "Arthur Verschaeve",
|
||||
"author_email": "contact@arthurverschaeve.be",
|
||||
"authored_date": "2015-02-01T21:56:31.000+01:00",
|
||||
"committer_name": "Arthur Verschaeve",
|
||||
"committer_email": "contact@arthurverschaeve.be",
|
||||
"committed_date": "2015-02-01T21:56:31.000+01:00"
|
||||
},
|
||||
"release": None,
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/foo%2Fbar/repository/commits/60a8ff033665e1207714d6670fcd7b65304ec02f$')
|
||||
def commit_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
|
||||
"short_id": "60a8ff03366",
|
||||
"title": "Sanitize for network graph",
|
||||
"author_name": "someguy",
|
||||
"author_email": "some.guy@gmail.com",
|
||||
"committer_name": "Some Guy",
|
||||
"committer_email": "some.guy@gmail.com",
|
||||
"created_at": "2012-09-20T09:06:12+03:00",
|
||||
"message": "Sanitize for network graph",
|
||||
"committed_date": "2012-09-20T09:06:12+03:00",
|
||||
"authored_date": "2012-09-20T09:06:12+03:00",
|
||||
"parent_ids": [
|
||||
"ae1d9fb46aa2b07ee9836d49862ec4e2c46fbbba"
|
||||
],
|
||||
"last_pipeline" : {
|
||||
"id": 8,
|
||||
"ref": "master",
|
||||
"sha": "2dc6aa325a317eda67812f05600bdf0fcdc70ab0",
|
||||
"status": "created",
|
||||
},
|
||||
"stats": {
|
||||
"additions": 15,
|
||||
"deletions": 10,
|
||||
"total": 25
|
||||
},
|
||||
"status": "running"
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/deploy_keys$', method='POST')
|
||||
def create_deploykey_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": 1,
|
||||
"title": "Public key",
|
||||
"key": "ssh-rsa some stuff",
|
||||
"created_at": "2013-10-02T10:12:29Z",
|
||||
"can_push": False,
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/hooks$', method='POST')
|
||||
def create_hook_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({
|
||||
"id": 1,
|
||||
"url": "http://example.com/hook",
|
||||
"project_id": 4,
|
||||
"push_events": True,
|
||||
"issues_events": True,
|
||||
"confidential_issues_events": True,
|
||||
"merge_requests_events": True,
|
||||
"tag_push_events": True,
|
||||
"note_events": True,
|
||||
"job_events": True,
|
||||
"pipeline_events": True,
|
||||
"wiki_page_events": True,
|
||||
"enable_ssl_verification": True,
|
||||
"created_at": "2012-10-12T17:04:47Z",
|
||||
}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/hooks/1$', method='DELETE')
|
||||
def delete_hook_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/deploy_keys/1$', method='DELETE')
|
||||
def delete_deploykey_handker(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps({}),
|
||||
}
|
||||
|
||||
|
||||
@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/users/knownuser/projects$')
|
||||
def user_projects_list_handler(_, request):
|
||||
if not request.headers.get('Authorization') == 'Bearer foobar':
|
||||
return {'status_code': 401}
|
||||
|
||||
return {
|
||||
'status_code': 200,
|
||||
'headers': {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
'content': json.dumps([
|
||||
{
|
||||
"id": 2,
|
||||
"name": "Another project",
|
||||
"description": None,
|
||||
"default_branch": "master",
|
||||
"visibility": "public",
|
||||
"path": "anotherproject",
|
||||
"path_with_namespace": "knownuser/anotherproject",
|
||||
"last_activity_at": "2013-09-30T13:46:02Z",
|
||||
"web_url": "http://example.com/knownuser/anotherproject",
|
||||
}
|
||||
]),
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_gitlab_trigger(dockerfile_path='', add_permissions=True, missing_avatar_url=False):
|
||||
handlers = [user_handler, users_handler, project_branches_handler, project_tree_handler,
|
||||
project_handler, get_projects_handler(add_permissions), tag_handler,
|
||||
project_branch_handler, get_group_handler(missing_avatar_url), dockerfile_handler,
|
||||
sub_dockerfile_handler, namespace_handler, user_namespace_handler, namespaces_handler,
|
||||
commit_handler, create_deploykey_handler, delete_deploykey_handker,
|
||||
create_hook_handler, delete_hook_handler, project_tags_handler,
|
||||
user_projects_list_handler, catchall_handler]
|
||||
|
||||
with HTTMock(*handlers):
|
||||
trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger'))
|
||||
trigger = GitLabBuildTrigger(trigger_obj, {
|
||||
'build_source': 'foo/bar',
|
||||
|
|
@ -12,208 +590,8 @@ def get_gitlab_trigger(dockerfile_path=''):
|
|||
'username': 'knownuser'
|
||||
})
|
||||
|
||||
trigger._get_authorized_client = get_mock_gitlab(with_nulls=False)
|
||||
return trigger
|
||||
client = gitlab.Gitlab('http://fakegitlab', oauth_token='foobar', timeout=20, api_version=4)
|
||||
client.auth()
|
||||
|
||||
def adddeploykey_mock(project_id, name, public_key):
|
||||
return {'id': 'foo'}
|
||||
|
||||
def addprojecthook_mock(project_id, webhook_url, push=False):
|
||||
return {'id': 'foo'}
|
||||
|
||||
def get_currentuser_mock():
|
||||
return {
|
||||
'username': 'knownuser'
|
||||
}
|
||||
|
||||
def project(namespace, name, is_org=False):
|
||||
project_access = None
|
||||
|
||||
if name != 'null':
|
||||
if namespace == 'knownuser':
|
||||
project_access = {
|
||||
'access_level': 50,
|
||||
}
|
||||
else:
|
||||
project_access = {
|
||||
'access_level': 0,
|
||||
}
|
||||
|
||||
data = {
|
||||
'id': '%s/%s' % (namespace, name),
|
||||
'default_branch': 'master',
|
||||
'namespace': {
|
||||
'id': namespace,
|
||||
'path': namespace,
|
||||
'name': namespace,
|
||||
},
|
||||
'path': name,
|
||||
'path_with_namespace': '%s/%s' % (namespace, name),
|
||||
'description': 'some %s repo' % name,
|
||||
'last_activity_at': str(datetime.utcfromtimestamp(0)),
|
||||
'web_url': 'https://bitbucket.org/%s/%s' % (namespace, name),
|
||||
'ssh_url_to_repo': 'git://%s/%s' % (namespace, name),
|
||||
'public': name != 'somerepo',
|
||||
'permissions': {
|
||||
'project_access': project_access,
|
||||
'group_access': {'access_level': 0},
|
||||
},
|
||||
'owner': {
|
||||
'avatar_url': 'avatarurl',
|
||||
}
|
||||
}
|
||||
|
||||
if name == 'null':
|
||||
del data['owner']['avatar_url']
|
||||
data['namespace']['avatar'] = None
|
||||
elif is_org:
|
||||
del data['owner']['avatar_url']
|
||||
data['namespace']['avatar'] = {'url': 'avatarurl'}
|
||||
|
||||
return data
|
||||
|
||||
def getprojects_mock(with_nulls=False):
|
||||
if with_nulls:
|
||||
def _getprojs(page=1, per_page=100):
|
||||
return [
|
||||
project('someorg', 'null', is_org=True),
|
||||
]
|
||||
return _getprojs
|
||||
|
||||
else:
|
||||
def _getprojs(page=1, per_page=100):
|
||||
return [
|
||||
project('knownuser', 'somerepo'),
|
||||
project('someorg', 'somerepo', is_org=True),
|
||||
project('someorg', 'anotherrepo', is_org=True),
|
||||
]
|
||||
return _getprojs
|
||||
|
||||
def getproject_mock(project_name):
|
||||
if project_name == 'knownuser/somerepo':
|
||||
return project('knownuser', 'somerepo')
|
||||
|
||||
if project_name == 'foo/bar':
|
||||
return project('foo', 'bar', is_org=True)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def getbranches_mock(project_id):
|
||||
return [
|
||||
{
|
||||
'name': 'master',
|
||||
'commit': {
|
||||
'id': 'aaaaaaa',
|
||||
}
|
||||
},
|
||||
{
|
||||
'name': 'otherbranch',
|
||||
'commit': {
|
||||
'id': 'aaaaaaa',
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
def getrepositorytags_mock(project_id):
|
||||
return [
|
||||
{
|
||||
'name': 'sometag',
|
||||
'commit': {
|
||||
'id': 'aaaaaaa',
|
||||
}
|
||||
},
|
||||
{
|
||||
'name': 'someothertag',
|
||||
'commit': {
|
||||
'id': 'aaaaaaa',
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
def getrepositorytree_mock(project_id, ref_name='master'):
|
||||
return [
|
||||
{'name': 'README'},
|
||||
{'name': 'Dockerfile'},
|
||||
]
|
||||
|
||||
def getrepositorycommit_mock(project_id, commit_sha):
|
||||
if commit_sha != 'aaaaaaa':
|
||||
return False
|
||||
|
||||
return {
|
||||
'id': 'aaaaaaa',
|
||||
'message': 'some message',
|
||||
'committed_date': 'now',
|
||||
}
|
||||
|
||||
def getusers_mock(search=None):
|
||||
if search == 'knownuser':
|
||||
return [
|
||||
{
|
||||
'username': 'knownuser',
|
||||
'avatar_url': 'avatarurl',
|
||||
}
|
||||
]
|
||||
|
||||
return False
|
||||
|
||||
def getbranch_mock(repo_id, branch):
|
||||
if branch != 'master' and branch != 'otherbranch':
|
||||
return False
|
||||
|
||||
return {
|
||||
'name': branch,
|
||||
'commit': {
|
||||
'id': 'aaaaaaa',
|
||||
}
|
||||
}
|
||||
|
||||
def gettag_mock(repo_id, tag):
|
||||
if tag != 'sometag' and tag != 'someothertag':
|
||||
return False
|
||||
|
||||
return {
|
||||
'name': tag,
|
||||
'commit': {
|
||||
'id': 'aaaaaaa',
|
||||
}
|
||||
}
|
||||
|
||||
def getrawfile_mock(repo_id, branch_name, path):
|
||||
if path == 'Dockerfile':
|
||||
return 'hello world'
|
||||
|
||||
if path == 'somesubdir/Dockerfile':
|
||||
return 'hi universe'
|
||||
|
||||
return False
|
||||
|
||||
def get_mock_gitlab(with_nulls=False):
|
||||
def _get_mock():
|
||||
mock_gitlab = Mock()
|
||||
mock_gitlab.host = 'https://bitbucket.org'
|
||||
|
||||
mock_gitlab.currentuser = Mock(side_effect=get_currentuser_mock)
|
||||
mock_gitlab.getusers = Mock(side_effect=getusers_mock)
|
||||
|
||||
mock_gitlab.getprojects = Mock(side_effect=getprojects_mock(with_nulls))
|
||||
mock_gitlab.getproject = Mock(side_effect=getproject_mock)
|
||||
mock_gitlab.getbranches = Mock(side_effect=getbranches_mock)
|
||||
|
||||
mock_gitlab.getbranch = Mock(side_effect=getbranch_mock)
|
||||
mock_gitlab.gettag = Mock(side_effect=gettag_mock)
|
||||
|
||||
mock_gitlab.getrepositorytags = Mock(side_effect=getrepositorytags_mock)
|
||||
mock_gitlab.getrepositorytree = Mock(side_effect=getrepositorytree_mock)
|
||||
mock_gitlab.getrepositorycommit = Mock(side_effect=getrepositorycommit_mock)
|
||||
|
||||
mock_gitlab.getrawfile = Mock(side_effect=getrawfile_mock)
|
||||
|
||||
mock_gitlab.adddeploykey = Mock(side_effect=adddeploykey_mock)
|
||||
mock_gitlab.addprojecthook = Mock(side_effect=addprojecthook_mock)
|
||||
mock_gitlab.deletedeploykey = Mock(return_value=True)
|
||||
mock_gitlab.deleteprojecthook = Mock(return_value=True)
|
||||
return mock_gitlab
|
||||
|
||||
return _get_mock
|
||||
trigger._get_authorized_client = lambda: client
|
||||
yield trigger
|
||||
|
|
|
|||
|
|
@ -3,12 +3,11 @@ import pytest
|
|||
from buildtrigger.triggerutil import TriggerStartException
|
||||
from buildtrigger.test.bitbucketmock import get_bitbucket_trigger
|
||||
from buildtrigger.test.githubmock import get_github_trigger
|
||||
from buildtrigger.test.gitlabmock import get_gitlab_trigger
|
||||
from endpoints.building import PreparedBuild
|
||||
|
||||
# Note: This test suite executes a common set of tests against all the trigger types specified
|
||||
# in this fixture. Each trigger's mock is expected to return the same data for all of these calls.
|
||||
@pytest.fixture(params=[get_github_trigger(), get_bitbucket_trigger(), get_gitlab_trigger()])
|
||||
@pytest.fixture(params=[get_github_trigger(), get_bitbucket_trigger()])
|
||||
def githost_trigger(request):
|
||||
return request.param
|
||||
|
||||
|
|
|
|||
|
|
@ -3,19 +3,20 @@ import pytest
|
|||
|
||||
from mock import Mock
|
||||
|
||||
from buildtrigger.test.gitlabmock import get_gitlab_trigger, get_mock_gitlab
|
||||
from buildtrigger.test.gitlabmock import get_gitlab_trigger
|
||||
from buildtrigger.triggerutil import (SkipRequestException, ValidationRequestException,
|
||||
InvalidPayloadException)
|
||||
InvalidPayloadException, TriggerStartException)
|
||||
from endpoints.building import PreparedBuild
|
||||
from util.morecollections import AttrDict
|
||||
|
||||
@pytest.fixture
|
||||
@pytest.fixture()
|
||||
def gitlab_trigger():
|
||||
return get_gitlab_trigger()
|
||||
with get_gitlab_trigger() as t:
|
||||
yield t
|
||||
|
||||
|
||||
def test_list_build_subdirs(gitlab_trigger):
|
||||
assert gitlab_trigger.list_build_subdirs() == ['/Dockerfile']
|
||||
assert gitlab_trigger.list_build_subdirs() == ['Dockerfile']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('dockerfile_path, contents', [
|
||||
|
|
@ -24,7 +25,7 @@ def test_list_build_subdirs(gitlab_trigger):
|
|||
('unknownpath', None),
|
||||
])
|
||||
def test_load_dockerfile_contents(dockerfile_path, contents):
|
||||
trigger = get_gitlab_trigger(dockerfile_path)
|
||||
with get_gitlab_trigger(dockerfile_path=dockerfile_path) as trigger:
|
||||
assert trigger.load_dockerfile_contents() == contents
|
||||
|
||||
|
||||
|
|
@ -37,23 +38,47 @@ def test_lookup_user(email, expected_response, gitlab_trigger):
|
|||
assert gitlab_trigger.lookup_user(email) == expected_response
|
||||
|
||||
|
||||
def test_null_permissions(gitlab_trigger):
|
||||
gitlab_trigger._get_authorized_client = get_mock_gitlab(with_nulls=True)
|
||||
sources = gitlab_trigger.list_build_sources_for_namespace('someorg')
|
||||
def test_null_permissions():
|
||||
with get_gitlab_trigger(add_permissions=False) as trigger:
|
||||
sources = trigger.list_build_sources_for_namespace('someorg')
|
||||
source = sources[0]
|
||||
assert source['has_admin_permissions']
|
||||
|
||||
|
||||
def test_null_avatar(gitlab_trigger):
|
||||
gitlab_trigger._get_authorized_client = get_mock_gitlab(with_nulls=True)
|
||||
namespace_data = gitlab_trigger.list_build_source_namespaces()
|
||||
def test_list_build_sources():
|
||||
with get_gitlab_trigger() as trigger:
|
||||
sources = trigger.list_build_sources_for_namespace('someorg')
|
||||
assert sources == [
|
||||
{
|
||||
'last_updated': 1380548762,
|
||||
'name': u'someproject',
|
||||
'url': u'http://example.com/someorg/someproject',
|
||||
'private': True,
|
||||
'full_name': u'someorg/someproject',
|
||||
'has_admin_permissions': False,
|
||||
'description': ''
|
||||
},
|
||||
{
|
||||
'last_updated': 1380548762,
|
||||
'name': u'anotherproject',
|
||||
'url': u'http://example.com/someorg/anotherproject',
|
||||
'private': False,
|
||||
'full_name': u'someorg/anotherproject',
|
||||
'has_admin_permissions': True,
|
||||
'description': '',
|
||||
}]
|
||||
|
||||
|
||||
def test_null_avatar():
|
||||
with get_gitlab_trigger(missing_avatar_url=True) as trigger:
|
||||
namespace_data = trigger.list_build_source_namespaces()
|
||||
expected = {
|
||||
'avatar_url': None,
|
||||
'personal': False,
|
||||
'title': 'someorg',
|
||||
'url': 'https://bitbucket.org/someorg',
|
||||
'title': u'someorg',
|
||||
'url': u'http://gitlab.com/groups/someorg',
|
||||
'score': 1,
|
||||
'id': 'someorg',
|
||||
'id': '2',
|
||||
}
|
||||
|
||||
assert namespace_data == [expected]
|
||||
|
|
@ -112,3 +137,95 @@ def test_handle_trigger_request(gitlab_trigger, payload, expected_error, expecte
|
|||
assert isinstance(gitlab_trigger.handle_trigger_request(request), PreparedBuild)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('run_parameters, expected_error, expected_message', [
|
||||
# No branch or tag specified: use the commit of the default branch.
|
||||
({}, None, None),
|
||||
|
||||
# Invalid branch.
|
||||
({'refs': {'kind': 'branch', 'name': 'invalid'}}, TriggerStartException,
|
||||
'Could not find branch in repository'),
|
||||
|
||||
# Invalid tag.
|
||||
({'refs': {'kind': 'tag', 'name': 'invalid'}}, TriggerStartException,
|
||||
'Could not find tag in repository'),
|
||||
|
||||
# Valid branch.
|
||||
({'refs': {'kind': 'branch', 'name': 'master'}}, None, None),
|
||||
|
||||
# Valid tag.
|
||||
({'refs': {'kind': 'tag', 'name': 'sometag'}}, None, None),
|
||||
])
|
||||
def test_manual_start(run_parameters, expected_error, expected_message, gitlab_trigger):
|
||||
if expected_error is not None:
|
||||
with pytest.raises(expected_error) as ipe:
|
||||
gitlab_trigger.manual_start(run_parameters)
|
||||
assert ipe.value.message == expected_message
|
||||
else:
|
||||
assert isinstance(gitlab_trigger.manual_start(run_parameters), PreparedBuild)
|
||||
|
||||
|
||||
def test_activate_and_deactivate(gitlab_trigger):
|
||||
_, private_key = gitlab_trigger.activate('http://some/url')
|
||||
assert 'private_key' in private_key
|
||||
|
||||
gitlab_trigger.deactivate()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('name, expected', [
|
||||
('refs', [
|
||||
{'kind': 'branch', 'name': 'master'},
|
||||
{'kind': 'branch', 'name': 'otherbranch'},
|
||||
{'kind': 'tag', 'name': 'sometag'},
|
||||
{'kind': 'tag', 'name': 'someothertag'},
|
||||
]),
|
||||
('tag_name', set(['sometag', 'someothertag'])),
|
||||
('branch_name', set(['master', 'otherbranch'])),
|
||||
('invalid', None)
|
||||
])
|
||||
def test_list_field_values(name, expected, gitlab_trigger):
|
||||
if expected is None:
|
||||
assert gitlab_trigger.list_field_values(name) is None
|
||||
elif isinstance(expected, set):
|
||||
assert set(gitlab_trigger.list_field_values(name)) == set(expected)
|
||||
else:
|
||||
assert gitlab_trigger.list_field_values(name) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('namespace, expected', [
|
||||
('', []),
|
||||
('unknown', []),
|
||||
|
||||
('knownuser', [
|
||||
{
|
||||
'last_updated': 1380548762,
|
||||
'name': u'anotherproject',
|
||||
'url': u'http://example.com/knownuser/anotherproject',
|
||||
'private': False,
|
||||
'full_name': u'knownuser/anotherproject',
|
||||
'has_admin_permissions': True,
|
||||
'description': ''
|
||||
},
|
||||
]),
|
||||
|
||||
('someorg', [
|
||||
{
|
||||
'last_updated': 1380548762,
|
||||
'name': u'someproject',
|
||||
'url': u'http://example.com/someorg/someproject',
|
||||
'private': True,
|
||||
'full_name': u'someorg/someproject',
|
||||
'has_admin_permissions': False,
|
||||
'description': ''
|
||||
},
|
||||
{
|
||||
'last_updated': 1380548762,
|
||||
'name': u'anotherproject',
|
||||
'url': u'http://example.com/someorg/anotherproject',
|
||||
'private': False,
|
||||
'full_name': u'someorg/anotherproject',
|
||||
'has_admin_permissions': True,
|
||||
'description': '',
|
||||
}]),
|
||||
])
|
||||
def test_list_build_sources_for_namespace(namespace, expected, gitlab_trigger):
|
||||
assert gitlab_trigger.list_build_sources_for_namespace(namespace) == expected
|
||||
|
|
|
|||
|
|
@ -3,37 +3,43 @@ import io
|
|||
import logging
|
||||
import re
|
||||
|
||||
class InvalidPayloadException(Exception):
|
||||
class TriggerException(Exception):
|
||||
pass
|
||||
|
||||
class BuildArchiveException(Exception):
|
||||
class TriggerAuthException(TriggerException):
|
||||
pass
|
||||
|
||||
class InvalidServiceException(Exception):
|
||||
class InvalidPayloadException(TriggerException):
|
||||
pass
|
||||
|
||||
class TriggerActivationException(Exception):
|
||||
class BuildArchiveException(TriggerException):
|
||||
pass
|
||||
|
||||
class TriggerDeactivationException(Exception):
|
||||
class InvalidServiceException(TriggerException):
|
||||
pass
|
||||
|
||||
class TriggerStartException(Exception):
|
||||
class TriggerActivationException(TriggerException):
|
||||
pass
|
||||
|
||||
class ValidationRequestException(Exception):
|
||||
class TriggerDeactivationException(TriggerException):
|
||||
pass
|
||||
|
||||
class SkipRequestException(Exception):
|
||||
class TriggerStartException(TriggerException):
|
||||
pass
|
||||
|
||||
class EmptyRepositoryException(Exception):
|
||||
class ValidationRequestException(TriggerException):
|
||||
pass
|
||||
|
||||
class RepositoryReadException(Exception):
|
||||
class SkipRequestException(TriggerException):
|
||||
pass
|
||||
|
||||
class TriggerProviderException(Exception):
|
||||
class EmptyRepositoryException(TriggerException):
|
||||
pass
|
||||
|
||||
class RepositoryReadException(TriggerException):
|
||||
pass
|
||||
|
||||
class TriggerProviderException(TriggerException):
|
||||
pass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -11,9 +11,7 @@ from app import app
|
|||
from auth.permissions import (UserAdminPermission, AdministerOrganizationPermission,
|
||||
ReadRepositoryPermission, AdministerRepositoryPermission)
|
||||
from buildtrigger.basehandler import BuildTriggerHandler
|
||||
from buildtrigger.triggerutil import (TriggerDeactivationException,
|
||||
TriggerActivationException, EmptyRepositoryException,
|
||||
RepositoryReadException, TriggerStartException)
|
||||
from buildtrigger.triggerutil import TriggerException, EmptyRepositoryException
|
||||
from data import model
|
||||
from data.model.build import update_build_trigger
|
||||
from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin,
|
||||
|
|
@ -118,7 +116,7 @@ class BuildTrigger(RepositoryParamResource):
|
|||
if handler.is_active():
|
||||
try:
|
||||
handler.deactivate()
|
||||
except TriggerDeactivationException as ex:
|
||||
except TriggerException as ex:
|
||||
# We are just going to eat this error
|
||||
logger.warning('Trigger deactivation problem: %s', ex)
|
||||
|
||||
|
|
@ -178,7 +176,7 @@ class BuildTriggerSubdirs(RepositoryParamResource):
|
|||
'contextMap': {},
|
||||
'dockerfile_paths': [],
|
||||
}
|
||||
except RepositoryReadException as exc:
|
||||
except TriggerException as exc:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': exc.message,
|
||||
|
|
@ -264,7 +262,7 @@ class BuildTriggerActivate(RepositoryParamResource):
|
|||
if 'private_key' in private_config:
|
||||
trigger.private_key = private_config['private_key']
|
||||
|
||||
except TriggerActivationException as exc:
|
||||
except TriggerException as exc:
|
||||
write_token.delete_instance()
|
||||
raise request_error(message=exc.message)
|
||||
|
||||
|
|
@ -332,7 +330,7 @@ class BuildTriggerAnalyze(RepositoryParamResource):
|
|||
new_config_dict,
|
||||
AdministerOrganizationPermission(namespace_name).can())
|
||||
return trigger_analyzer.analyze_trigger()
|
||||
except RepositoryReadException as rre:
|
||||
except TriggerException as rre:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': 'Could not analyze the repository: %s' % rre.message,
|
||||
|
|
@ -391,7 +389,7 @@ class ActivateBuildTrigger(RepositoryParamResource):
|
|||
run_parameters = request.get_json()
|
||||
prepared = handler.manual_start(run_parameters=run_parameters)
|
||||
build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
|
||||
except TriggerStartException as tse:
|
||||
except TriggerException as tse:
|
||||
raise InvalidRequest(tse.message)
|
||||
except MaximumBuildsQueuedException:
|
||||
abort(429, message='Maximum queued build rate exceeded.')
|
||||
|
|
@ -494,7 +492,7 @@ class BuildTriggerSources(RepositoryParamResource):
|
|||
return {
|
||||
'sources': handler.list_build_sources_for_namespace(namespace)
|
||||
}
|
||||
except RepositoryReadException as rre:
|
||||
except TriggerException as rre:
|
||||
raise InvalidRequest(rre.message)
|
||||
else:
|
||||
raise Unauthorized()
|
||||
|
|
@ -522,7 +520,7 @@ class BuildTriggerSourceNamespaces(RepositoryParamResource):
|
|||
return {
|
||||
'namespaces': handler.list_build_source_namespaces()
|
||||
}
|
||||
except RepositoryReadException as rre:
|
||||
except TriggerException as rre:
|
||||
raise InvalidRequest(rre.message)
|
||||
else:
|
||||
raise Unauthorized()
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
-e git+https://github.com/NateFerrero/oauth2lib.git#egg=oauth2lib
|
||||
-e git+https://github.com/coreos/mockldap.git@v0.1.x#egg=mockldap
|
||||
-e git+https://github.com/coreos/py-bitbucket.git#egg=py-bitbucket
|
||||
-e git+https://github.com/coreos/pyapi-gitlab.git@timeout#egg=pyapi-gitlab
|
||||
-e git+https://github.com/coreos/resumablehashlib.git#egg=resumablehashlib
|
||||
-e git+https://github.com/jepcastelein/marketo-rest-python.git#egg=marketorestpython
|
||||
-e git+https://github.com/app-registry/appr-server.git@c2ef3b88afe926a92ef5f2e11e7d4a259e286a17#egg=cnr_server # naming has changed
|
||||
|
|
@ -58,6 +57,7 @@ pyjwkest
|
|||
pyjwt
|
||||
pymysql==0.6.7 # Remove version when baseimage has Python 2.7.9+
|
||||
python-dateutil
|
||||
python-gitlab
|
||||
python-keystoneclient
|
||||
python-ldap
|
||||
python-magic
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ aiowsgi==0.6
|
|||
alembic==0.9.8
|
||||
-e git+https://github.com/coreos/mockldap.git@59a46efbe8c7cd8146a87a7c4f2b09746b953e11#egg=mockldap
|
||||
-e git+https://github.com/coreos/py-bitbucket.git@55a1ada645f2fb6369147996ec71edd7828d91c8#egg=py_bitbucket
|
||||
-e git+https://github.com/coreos/pyapi-gitlab.git@136c3970d591136a4f766a846c5d22aad52e124f#egg=pyapi_gitlab
|
||||
-e git+https://github.com/coreos/resumablehashlib.git@b1b631249589b07adf40e0ee545b323a501340b4#egg=resumablehashlib
|
||||
-e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601
|
||||
-e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode
|
||||
|
|
@ -113,6 +112,7 @@ pyparsing==2.2.0
|
|||
PyPDF2==1.26.0
|
||||
python-dateutil==2.6.1
|
||||
python-editor==1.0.3
|
||||
python-gitlab==1.4.0
|
||||
python-keystoneclient==3.15.0
|
||||
python-ldap==2.5.2
|
||||
python-magic==0.4.15
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@
|
|||
<img class="namespace-avatar" ng-src="{{ ::item.avatar_url }}" ng-if="::item.avatar_url">
|
||||
<span class="anchor"
|
||||
href="{{ ::item.url }}"
|
||||
is-text-only="::!item.url">{{ ::item.id }}</span>
|
||||
is-text-only="::!item.url">{{ ::item.title }}</span>
|
||||
</script>
|
||||
</cor-table-col>
|
||||
<cor-table-col title="Importance"
|
||||
|
|
|
|||
Reference in a new issue