Merge pull request #3110 from quay/joseph.schorr/QUAY-966/gitlab-v4

Reimplement GitLab trigger handler using the V4 API library

buildtrigger/gitlabhandler.py

@@ -1,28 +1,26 @@
+import os.path
 import logging
-import os
 
 from calendar import timegm
 from functools import wraps
 
 import dateutil.parser
-
+import gitlab
-from app import app, gitlab_trigger
+import requests
 
 from jsonschema import validate
+
+from app import app, gitlab_trigger
 from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
                                       TriggerDeactivationException, TriggerStartException,
                                       SkipRequestException, InvalidPayloadException,
+                                      TriggerAuthException,
                                       determine_build_ref, raise_if_skipped_build,
                                       find_matching_branches)
-
 from buildtrigger.basehandler import BuildTriggerHandler
-
+from endpoints.exception import ExternalServiceError
 from util.security.ssh import generate_ssh_keypair
 from util.dict_wrappers import JSONPathDict, SafeDictSetter
-from endpoints.exception import ExternalServiceError
-
-import gitlab
-import requests
 
 logger = logging.getLogger(__name__)
 
@@ -89,7 +87,7 @@ _ACCESS_LEVEL_MAP = {
 _PER_PAGE_COUNT = 20
 
 
-def _catch_timeouts(func):
+def _catch_timeouts_and_errors(func):
   @wraps(func)
   def wrapper(*args, **kwargs):
     try:
@@ -98,17 +96,21 @@ def _catch_timeouts(func):
       msg = 'Request to the GitLab API timed out'
       logger.exception(msg)
       raise ExternalServiceError(msg)
+    except gitlab.GitlabError:
+      msg = 'GitLab API error. Please contact support.'
+      logger.exception(msg)
+      raise ExternalServiceError(msg)      
   return wrapper
 
 
-def _paginated_iterator(func, exc):
+def _paginated_iterator(func, exc, **kwargs):
   """ Returns an iterator over invocations of the given function, automatically handling
       pagination.
   """
-  page = 0
+  page = 1
   while True:
-    result = func(page=page, per_page=_PER_PAGE_COUNT)
+    result = func(page=page, per_page=_PER_PAGE_COUNT, **kwargs)
-    if result is False:
+    if result is None or result is False:
       raise exc
 
     counter = 0
@@ -196,20 +198,28 @@ class GitLabBuildTrigger(BuildTriggerHandler):
 
   def _get_authorized_client(self):
     auth_token = self.auth_token or 'invalid'
-    return gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=5)
+    api_version = self.config.get('API_VERSION', '4')
+    client = gitlab.Gitlab(gitlab_trigger.api_endpoint(), oauth_token=auth_token, timeout=20,
+                           api_version=api_version)
+    try:
+      client.auth()
+    except gitlab.GitlabGetError as ex:
+      raise TriggerAuthException(ex.message)
+
+    return client
 
   def is_active(self):
     return 'hook_id' in self.config
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def activate(self, standard_webhook_url):
     config = self.config
     new_build_source = config['build_source']
     gl_client = self._get_authorized_client()
 
     # Find the GitLab repository.
-    repository = gl_client.getproject(new_build_source)
+    gl_project = gl_client.projects.get(new_build_source)
-    if repository is False:
+    if not gl_project:
       msg = 'Unable to find GitLab repository for source: %s' % new_build_source
       raise TriggerActivationException(msg)
 
@@ -221,20 +231,28 @@ class GitLabBuildTrigger(BuildTriggerHandler):
         'value': public_key,
       },
     ]
-    key = gl_client.adddeploykey(repository['id'], '%s Builder' % app.config['REGISTRY_TITLE'],
+
-                                 public_key)
+    key = gl_project.keys.create({
-    if key is False:
+      'title': '%s Builder' % app.config['REGISTRY_TITLE'],
+      'key': public_key,
+    })
+
+    if not key:
       msg = 'Unable to add deploy key to repository: %s' % new_build_source
       raise TriggerActivationException(msg)
-    config['key_id'] = key['id']
+
+    config['key_id'] = key.get_id()
 
     # Add the webhook to the GitLab repository.
-    hook = gl_client.addprojecthook(repository['id'], standard_webhook_url, push=True)
+    hook = gl_project.hooks.create({
-    if hook is False:
+      'url': standard_webhook_url,
+      'push': True,
+    })
+    if not hook:
       msg = 'Unable to create webhook on repository: %s' % new_build_source
       raise TriggerActivationException(msg)
 
-    config['hook_id'] = hook['id']
+    config['hook_id'] = hook.get_id()
     self.config = config
     return config, {'private_key': private_key}
 
@@ -243,72 +261,71 @@ class GitLabBuildTrigger(BuildTriggerHandler):
     gl_client = self._get_authorized_client()
 
     # Find the GitLab repository.
-    repository = gl_client.getproject(config['build_source'])
+    gl_project = gl_client.projects.get(config['build_source'])
-    if repository is False:
+    if not gl_project:
       msg = 'Unable to find GitLab repository for source: %s' % config['build_source']
       raise TriggerDeactivationException(msg)
 
     # Remove the webhook.
-    success = gl_client.deleteprojecthook(repository['id'], config['hook_id'])
+    gl_project.hooks.delete(config['hook_id'])
-    if success is False:
-      msg = 'Unable to remove hook: %s' % config['hook_id']
-      raise TriggerDeactivationException(msg)
     config.pop('hook_id', None)
 
     # Remove the key
-    success = gl_client.deletedeploykey(repository['id'], config['key_id'])
+    gl_project.keys.delete(config['key_id'])
-    if success is False:
-      msg = 'Unable to remove deploy key: %s' % config['key_id']
-      raise TriggerDeactivationException(msg)
     config.pop('key_id', None)
 
     self.config = config
     return config
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def list_build_source_namespaces(self):
     gl_client = self._get_authorized_client()
-    current_user = gl_client.currentuser()
+    current_user = gl_client.user
-    if current_user is False:
+    if not current_user:
       raise RepositoryReadException('Unable to get current user')
 
     namespaces = {}
-    repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
+    for namespace in _paginated_iterator(gl_client.namespaces.list, RepositoryReadException):
-    for repo in repositories:
+      namespace_id = namespace.get_id()
-      namespace = repo.get('namespace') or {}
+
-      if not namespace:
+      # Retrieve the namespace as a user or group.
+      namespace_obj = self._get_namespace(gl_client, namespace)
+      if namespace_obj is None:
+        logger.warning('Could not load details for namespace %s', namespace_id)
         continue
 
-      namespace_id = namespace['id']
-
-      avatar_url = ''
-      if 'avatar' in namespace:
-        avatar_data = namespace.get('avatar') or {}
-        avatar_url = avatar_data.get('url')
-      elif 'owner' in repo:
-        owner_data = repo.get('owner') or {}
-        avatar_url = owner_data.get('avatar_url')
-
       if namespace_id in namespaces:
         namespaces[namespace_id]['score'] = namespaces[namespace_id]['score'] + 1
       else:
-        owner = namespace['name']
+        owner = namespace.attributes['name']
         namespaces[namespace_id] = {
-          'personal': owner == current_user['username'],
+          'personal': owner == current_user.attributes['username'],
-          'id': namespace['path'],
+          'id': str(namespace_id),
-          'title': namespace['name'],
+          'title': namespace.attributes['name'],
-          'avatar_url': avatar_url,
+          'avatar_url': namespace_obj.attributes.get('avatar_url', ''),
           'score': 1,
-          'url': gl_client.host + '/' + namespace['path'],
+          'url': namespace_obj.attributes.get('web_url', ''),
         }
 
     return BuildTriggerHandler.build_namespaces_response(namespaces)
 
-  @_catch_timeouts
+  def _get_namespace(self, gl_client, gl_namespace, lazy=False):
-  def list_build_sources_for_namespace(self, namespace):
+    try:
+      if gl_namespace.attributes['kind'] == 'group':
+        return gl_client.groups.get(gl_namespace.attributes['name'], lazy=lazy)
+
+      return gl_client.users.get(gl_namespace.attributes['name'], lazy=lazy)
+    except gitlab.GitlabGetError:
+      return None
+
+  @_catch_timeouts_and_errors
+  def list_build_sources_for_namespace(self, namespace_id):
+    if not namespace_id:
+      return []
+
     def repo_view(repo):
       # Because *anything* can be None in GitLab API!
-      permissions = repo.get('permissions') or {}
+      permissions = repo.attributes.get('permissions') or {}
       group_access = permissions.get('group_access') or {}
       project_access = permissions.get('project_access') or {}
 
@@ -327,17 +344,17 @@ class GitLabBuildTrigger(BuildTriggerHandler):
         has_admin_permission = True
 
       view = {
-        'name': repo['path'],
+        'name': repo.attributes['path'],
-        'full_name': repo['path_with_namespace'],
+        'full_name': repo.attributes['path_with_namespace'],
-        'description': repo.get('description') or '',
+        'description': repo.attributes.get('description') or '',
-        'url': repo.get('web_url'),
+        'url': repo.attributes.get('web_url'),
         'has_admin_permissions': has_admin_permission,
-        'private': repo.get('public', False) is False,
+        'private': repo.attributes.get('visibility') == 'private',
       }
 
-      if repo.get('last_activity_at'):
+      if repo.attributes.get('last_activity_at'):
         try:
-          last_modified = dateutil.parser.parse(repo['last_activity_at'])
+          last_modified = dateutil.parser.parse(repo.attributes['last_activity_at'])
           view['last_updated'] = timegm(last_modified.utctimetuple())
         except ValueError:
           logger.exception('Gitlab gave us an invalid last_activity_at: %s', last_modified)
@@ -345,44 +362,50 @@ class GitLabBuildTrigger(BuildTriggerHandler):
       return view
 
     gl_client = self._get_authorized_client()
-    repositories = _paginated_iterator(gl_client.getprojects, RepositoryReadException)
-    repos = [repo_view(repo) for repo in repositories if repo['namespace']['path'] == namespace]
-    return BuildTriggerHandler.build_sources_response(repos)
 
-  @_catch_timeouts
+    try:
+      gl_namespace = gl_client.namespaces.get(namespace_id)
+    except gitlab.GitlabGetError:
+      return []
+
+    namespace_obj = self._get_namespace(gl_client, gl_namespace, lazy=True)
+    repositories = _paginated_iterator(namespace_obj.projects.list, RepositoryReadException)
+    return BuildTriggerHandler.build_sources_response([repo_view(repo) for repo in repositories])
+
+  @_catch_timeouts_and_errors
   def list_build_subdirs(self):
     config = self.config
     gl_client = self._get_authorized_client()
     new_build_source = config['build_source']
 
-    repository = gl_client.getproject(new_build_source)
+    gl_project = gl_client.projects.get(new_build_source)
-    if repository is False:
+    if not gl_project:
       msg = 'Unable to find GitLab repository for source: %s' % new_build_source
       raise RepositoryReadException(msg)
 
-    repo_branches = gl_client.getbranches(repository['id'])
+    repo_branches = gl_project.branches.list()
-    if repo_branches is False:
+    if not repo_branches:
       msg = 'Unable to find GitLab branches for source: %s' % new_build_source
       raise RepositoryReadException(msg)
 
-    branches = [branch['name'] for branch in repo_branches]
+    branches = [branch.attributes['name'] for branch in repo_branches]
     branches = find_matching_branches(config, branches)
-    branches = branches or [repository['default_branch'] or 'master']
+    branches = branches or [gl_project.attributes['default_branch'] or 'master']
 
-    repo_tree = gl_client.getrepositorytree(repository['id'], ref_name=branches[0])
+    repo_tree = gl_project.repository_tree(ref=branches[0])
-    if repo_tree is False:
+    if not repo_tree:
       msg = 'Unable to find GitLab repository tree for source: %s' % new_build_source
       raise RepositoryReadException(msg)
 
-    return ["/" + node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
+    return [node['name'] for node in repo_tree if self.filename_is_dockerfile(node['name'])]
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def load_dockerfile_contents(self):
     gl_client = self._get_authorized_client()
     path = self.get_dockerfile_path()
 
-    repository = gl_client.getproject(self.config['build_source'])
+    gl_project = gl_client.projects.get(self.config['build_source'])
-    if repository is False:
+    if not gl_project:
       return None
 
     branches = self.list_field_values('branch_name')
@@ -391,16 +414,15 @@ class GitLabBuildTrigger(BuildTriggerHandler):
       return None
 
     branch_name = branches[0]
-    if repository['default_branch'] in branches:
+    if gl_project.attributes['default_branch'] in branches:
-      branch_name = repository['default_branch']
+      branch_name = gl_project.attributes['default_branch']
 
-    contents = gl_client.getrawfile(repository['id'], branch_name, path)
+    try:
-    if contents is False:
+      return gl_project.files.get(path, branch_name).decode()
+    except gitlab.GitlabGetError:
       return None
 
-    return contents
+  @_catch_timeouts_and_errors
-
-  @_catch_timeouts
   def list_field_values(self, field_name, limit=None):
     if field_name == 'refs':
       branches = self.list_field_values('branch_name')
@@ -410,139 +432,138 @@ class GitLabBuildTrigger(BuildTriggerHandler):
               [{'kind': 'tag', 'name': t} for t in tags])
 
     gl_client = self._get_authorized_client()
-    repo = gl_client.getproject(self.config['build_source'])
+    gl_project = gl_client.projects.get(self.config['build_source'])
-    if repo is False:
+    if not gl_project:
       return []
 
     if field_name == 'tag_name':
-      tags = gl_client.getrepositorytags(repo['id'])
+      tags = gl_project.tags.list()
-      if tags is False:
+      if not tags:
         return []
 
       if limit:
         tags = tags[0:limit]
 
-      return [tag['name'] for tag in tags]
+      return [tag.attributes['name'] for tag in tags]
 
     if field_name == 'branch_name':
-      branches = gl_client.getbranches(repo['id'])
+      branches = gl_project.branches.list()
-      if branches is False:
+      if not branches:
         return []
 
       if limit:
         branches = branches[0:limit]
 
-      return [branch['name'] for branch in branches]
+      return [branch.attributes['name'] for branch in branches]
 
     return None
 
   def get_repository_url(self):
     return gitlab_trigger.get_public_url(self.config['build_source'])
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def lookup_commit(self, repo_id, commit_sha):
     if repo_id is None:
       return None
 
     gl_client = self._get_authorized_client()
-    commit = gl_client.getrepositorycommit(repo_id, commit_sha)
+    gl_project = gl_client.projects.get(self.config['build_source'], lazy=True)
-    if commit is False:
+    commit = gl_project.commits.get(commit_sha)
+    if not commit:
       return None
 
     return commit
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def lookup_user(self, email):
     gl_client = self._get_authorized_client()
     try:
-      result = gl_client.getusers(search=email)
+      result = gl_client.users.list(search=email)
-      if result is False:
+      if not result:
         return None
 
       [user] = result
       return {
-        'username': user['username'],
+        'username': user.attributes['username'],
-        'html_url': gl_client.host + '/' + user['username'],
+        'html_url': user.attributes['web_url'],
-        'avatar_url': user['avatar_url']
+        'avatar_url': user.attributes['avatar_url']
       }
     except ValueError:
       return None
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def get_metadata_for_commit(self, commit_sha, ref, repo):
-    gl_client = self._get_authorized_client()
+    commit = self.lookup_commit(repo.get_id(), commit_sha)
-    commit = gl_client.getrepositorycommit(repo['id'], commit_sha)
+    if commit is None:
+      return None
 
     metadata = {
-      'commit': commit['id'],
+      'commit': commit.attributes['id'],
       'ref': ref,
-      'default_branch': repo['default_branch'],
+      'default_branch': repo.attributes['default_branch'],
-      'git_url': repo['ssh_url_to_repo'],
+      'git_url': repo.attributes['ssh_url_to_repo'],
       'commit_info': {
-        'url': gl_client.host + '/' + repo['path_with_namespace'] + '/commit/' + commit['id'],
+        'url': os.path.join(repo.attributes['web_url'], 'commit', commit.attributes['id']),
-        'message': commit['message'],
+        'message': commit.attributes['message'],
-        'date': commit['committed_date'],
+        'date': commit.attributes['committed_date'],
       },
     }
 
     committer = None
-    if 'committer_email' in commit:
+    if 'committer_email' in commit.attributes:
-      committer = self.lookup_user(commit['committer_email'])
+      committer = self.lookup_user(commit.attributes['committer_email'])
 
     author = None
-    if 'author_email' in commit:
+    if 'author_email' in commit.attributes:
-      author = self.lookup_user(commit['author_email'])
+      author = self.lookup_user(commit.attributes['author_email'])
 
     if committer is not None:
       metadata['commit_info']['committer'] = {
         'username': committer['username'],
         'avatar_url': committer['avatar_url'],
-        'url': gl_client.host + '/' + committer['username'],
+        'url': committer.get('http_url', ''),
       }
 
     if author is not None:
       metadata['commit_info']['author'] = {
         'username': author['username'],
         'avatar_url': author['avatar_url'],
-        'url': gl_client.host + '/' + author['username']
+        'url': author.get('http_url', ''),
       }
 
     return metadata
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def manual_start(self, run_parameters=None):
     gl_client = self._get_authorized_client()
-
+    gl_project = gl_client.projects.get(self.config['build_source'])
-    repo = gl_client.getproject(self.config['build_source'])
+    if not gl_project:
-    if repo is False:
       raise TriggerStartException('Could not find repository')
 
     def get_tag_sha(tag_name):
-      tags = gl_client.getrepositorytags(repo['id'])
+      try:
-      if tags is False:
+        tag = gl_project.tags.get(tag_name)
+      except gitlab.GitlabGetError:
         raise TriggerStartException('Could not find tag in repository')
 
-      for tag in tags:
+      return tag.attributes['commit']['id']
-        if tag['name'] == tag_name:
-          return tag['commit']['id']
-
-      raise TriggerStartException('Could not find tag in repository')
 
     def get_branch_sha(branch_name):
-      branch = gl_client.getbranch(repo['id'], branch_name)
+      try:
-      if branch is False:
+        branch = gl_project.branches.get(branch_name)
+      except gitlab.GitlabGetError:
         raise TriggerStartException('Could not find branch in repository')
 
-      return branch['commit']['id']
+      return branch.attributes['commit']['id']
 
     # Find the branch or tag to build.
     (commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
-                                            repo['default_branch'])
+                                            gl_project.attributes['default_branch'])
 
-    metadata = self.get_metadata_for_commit(commit_sha, ref, repo)
+    metadata = self.get_metadata_for_commit(commit_sha, ref, gl_project)
     return self.prepare_build(metadata, is_manual=True)
 
-  @_catch_timeouts
+  @_catch_timeouts_and_errors
   def handle_trigger_request(self, request):
     payload = request.get_json()
     if not payload:
@@ -552,12 +573,12 @@ class GitLabBuildTrigger(BuildTriggerHandler):
 
     # Lookup the default branch.
     gl_client = self._get_authorized_client()
-    repo = gl_client.getproject(self.config['build_source'])
+    gl_project = gl_client.projects.get(self.config['build_source'])
-    if repo is False:
+    if not gl_project:
       logger.debug('Skipping GitLab build; project %s not found', self.config['build_source'])
       raise InvalidPayloadException()
 
-    default_branch = repo['default_branch']
+    default_branch = gl_project.attributes['default_branch']
     metadata = get_transformed_webhook_payload(payload, default_branch=default_branch,
                                                lookup_user=self.lookup_user,
                                                lookup_commit=self.lookup_commit)

buildtrigger/test/gitlabmock.py

@@ -1,10 +1,588 @@
-from datetime import datetime
+import base64
-from mock import Mock
+import json
+
+from contextlib import contextmanager
+
+import gitlab
+
+from httmock import urlmatch, HTTMock
 
 from buildtrigger.gitlabhandler import GitLabBuildTrigger
 from util.morecollections import AttrDict
 
-def get_gitlab_trigger(dockerfile_path=''):
+
+@urlmatch(netloc=r'fakegitlab')
+def catchall_handler(url, request):
+  return {'status_code': 404}
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/users$')
+def users_handler(url, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  if url.query.find('knownuser') < 0:
+    return {
+      'status_code': 200,
+      'headers': {
+        'Content-Type': 'application/json',
+      },
+      'content': json.dumps([]),
+    }
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps([
+      {
+        "id": 1,
+        "username": "knownuser",
+        "name": "Known User",
+        "state": "active",
+        "avatar_url": "avatarurl",
+        "web_url": "https://bitbucket.org/knownuser",
+      },
+    ]),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/user$')
+def user_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "id": 1,
+      "username": "john_smith",
+      "email": "john@example.com",
+      "name": "John Smith",
+      "state": "active",
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/foo%2Fbar$')
+def project_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "id": 4,
+      "description": None,
+      "default_branch": "master",
+      "visibility": "private",
+      "path_with_namespace": "someorg/somerepo",
+      "ssh_url_to_repo": "git@example.com:someorg/somerepo.git",
+      "web_url": "http://example.com/someorg/somerepo",
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tree$')
+def project_tree_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps([
+      {
+        "id": "a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
+        "name": "Dockerfile",
+        "type": "tree",
+        "path": "files/Dockerfile",
+        "mode": "040000",
+      },
+    ]),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tags$')
+def project_tags_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps([
+      {
+        'name': 'sometag',
+        'commit': {
+          'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
+        },
+      },
+      {
+        'name': 'someothertag',
+        'commit': {
+          'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
+        },
+      },
+    ]),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/branches$')
+def project_branches_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps([
+      {
+        'name': 'master',
+        'commit': {
+          'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
+        },
+      },
+      {
+        'name': 'otherbranch',
+        'commit': {
+          'id': '60a8ff033665e1207714d6670fcd7b65304ec02f',
+        },
+      },
+    ]),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/branches/master$')
+def project_branch_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "name": "master",
+      "merged": True,
+      "protected": True,
+      "developers_can_push": False,
+      "developers_can_merge": False,
+      "commit": {
+        "author_email": "john@example.com",
+        "author_name": "John Smith",
+        "authored_date": "2012-06-27T05:51:39-07:00",
+        "committed_date": "2012-06-28T03:44:20-07:00",
+        "committer_email": "john@example.com",
+        "committer_name": "John Smith",
+        "id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
+        "short_id": "7b5c3cc",
+        "title": "add projects API",
+        "message": "add projects API",
+        "parent_ids": [
+          "4ad91d3c1144c406e50c7b33bae684bd6837faf8",
+        ],
+      },
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces/someorg$')
+def namespace_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "id": 2,
+      "name": "someorg",
+      "path": "someorg",
+      "kind": "group",
+      "full_path": "someorg",
+      "parent_id": None,
+      "members_count_with_descendants": 2
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces/knownuser$')
+def user_namespace_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "id": 2,
+      "name": "knownuser",
+      "path": "knownuser",
+      "kind": "user",
+      "full_path": "knownuser",
+      "parent_id": None,
+      "members_count_with_descendants": 2
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/namespaces(/)?$')
+def namespaces_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps([{
+      "id": 2,
+      "name": "someorg",
+      "path": "someorg",
+      "kind": "group",
+      "full_path": "someorg",
+      "parent_id": None,
+      "members_count_with_descendants": 2
+    }]),
+  }
+
+
+def get_projects_handler(add_permissions_block):
+  @urlmatch(netloc=r'fakegitlab', path=r'/api/v4/groups/someorg/projects$')
+  def projects_handler(_, request):
+    if not request.headers.get('Authorization') == 'Bearer foobar':
+      return {'status_code': 401}
+
+    permissions_block = {
+      "project_access": {
+        "access_level": 10,
+        "notification_level": 3
+      },
+      "group_access": {
+        "access_level": 20,
+        "notification_level": 3
+      },
+    }
+
+    return {
+      'status_code': 200,
+      'headers': {
+        'Content-Type': 'application/json',
+      },
+      'content': json.dumps([{
+        "id": 4,
+        "name": "Some project",
+        "description": None,
+        "default_branch": "master",
+        "visibility": "private",
+        "path": "someproject",
+        "path_with_namespace": "someorg/someproject",
+        "last_activity_at": "2013-09-30T13:46:02Z",
+        "web_url": "http://example.com/someorg/someproject",
+        "permissions": permissions_block if add_permissions_block else None,
+      },
+      {
+        "id": 5,
+        "name": "Another project",
+        "description": None,
+        "default_branch": "master",
+        "visibility": "public",
+        "path": "anotherproject",
+        "path_with_namespace": "someorg/anotherproject",
+        "last_activity_at": "2013-09-30T13:46:02Z",
+        "web_url": "http://example.com/someorg/anotherproject",
+      }]),
+    }
+  return projects_handler
+
+
+def get_group_handler(null_avatar):
+  @urlmatch(netloc=r'fakegitlab', path=r'/api/v4/groups/someorg$')
+  def group_handler(_, request):
+    if not request.headers.get('Authorization') == 'Bearer foobar':
+      return {'status_code': 401}
+
+    return {
+      'status_code': 200,
+      'headers': {
+        'Content-Type': 'application/json',
+      },
+      'content': json.dumps({
+        "id": 1,
+        "name": "SomeOrg Group",
+        "path": "someorg",
+        "description": "An interesting group",
+        "visibility": "public",
+        "lfs_enabled": True,
+        "avatar_url": 'avatar_url' if not null_avatar else None,
+        "web_url": "http://gitlab.com/groups/someorg",
+        "request_access_enabled": False,
+        "full_name": "SomeOrg Group",
+        "full_path": "someorg",
+        "parent_id": None,
+      }),
+    }
+  return group_handler
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/files/Dockerfile$')
+def dockerfile_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "file_name": "Dockerfile",
+      "file_path": "Dockerfile",
+      "size": 10,
+      "encoding": "base64",
+      "content": base64.b64encode('hello world'),
+      "ref": "master",
+      "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
+      "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
+      "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d"
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/files/somesubdir%2FDockerfile$')
+def sub_dockerfile_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "file_name": "Dockerfile",
+      "file_path": "somesubdir/Dockerfile",
+      "size": 10,
+      "encoding": "base64",
+      "content": base64.b64encode('hi universe'),
+      "ref": "master",
+      "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83",
+      "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50",
+      "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d"
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/repository/tags/sometag$')
+def tag_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "name": "sometag",
+      "message": "some cool message",
+      "target": "60a8ff033665e1207714d6670fcd7b65304ec02f",
+      "commit": {
+        "id": "60a8ff033665e1207714d6670fcd7b65304ec02f",
+        "short_id": "60a8ff03",
+        "title": "Initial commit",
+        "created_at": "2017-07-26T11:08:53.000+02:00",
+        "parent_ids": [
+          "f61c062ff8bcbdb00e0a1b3317a91aed6ceee06b"
+        ],
+        "message": "v5.0.0\n",
+        "author_name": "Arthur Verschaeve",
+        "author_email": "contact@arthurverschaeve.be",
+        "authored_date": "2015-02-01T21:56:31.000+01:00",
+        "committer_name": "Arthur Verschaeve",
+        "committer_email": "contact@arthurverschaeve.be",
+        "committed_date": "2015-02-01T21:56:31.000+01:00"
+      },
+      "release": None,
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/foo%2Fbar/repository/commits/60a8ff033665e1207714d6670fcd7b65304ec02f$')
+def commit_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "id": "60a8ff033665e1207714d6670fcd7b65304ec02f",             
+      "short_id": "60a8ff03366",
+      "title": "Sanitize for network graph",
+      "author_name": "someguy",
+      "author_email": "some.guy@gmail.com",
+      "committer_name": "Some Guy",
+      "committer_email": "some.guy@gmail.com",
+      "created_at": "2012-09-20T09:06:12+03:00",
+      "message": "Sanitize for network graph",
+      "committed_date": "2012-09-20T09:06:12+03:00",
+      "authored_date": "2012-09-20T09:06:12+03:00",
+      "parent_ids": [
+        "ae1d9fb46aa2b07ee9836d49862ec4e2c46fbbba"
+      ],
+      "last_pipeline" : {
+        "id": 8,
+        "ref": "master",
+        "sha": "2dc6aa325a317eda67812f05600bdf0fcdc70ab0",
+        "status": "created",
+      },
+      "stats": {
+        "additions": 15,
+        "deletions": 10,
+        "total": 25
+      },
+      "status": "running"
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/deploy_keys$', method='POST')
+def create_deploykey_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "id": 1,
+      "title": "Public key",
+      "key": "ssh-rsa some stuff",
+      "created_at": "2013-10-02T10:12:29Z",
+      "can_push": False,
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/hooks$', method='POST')
+def create_hook_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({
+      "id": 1,
+      "url": "http://example.com/hook",
+      "project_id": 4,
+      "push_events": True,
+      "issues_events": True,
+      "confidential_issues_events": True,
+      "merge_requests_events": True,
+      "tag_push_events": True,
+      "note_events": True,
+      "job_events": True,
+      "pipeline_events": True,
+      "wiki_page_events": True,
+      "enable_ssl_verification": True,
+      "created_at": "2012-10-12T17:04:47Z",
+    }),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/hooks/1$', method='DELETE')
+def delete_hook_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({}),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/projects/4/deploy_keys/1$', method='DELETE')
+def delete_deploykey_handker(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps({}),
+  }
+
+
+@urlmatch(netloc=r'fakegitlab', path=r'/api/v4/users/knownuser/projects$')
+def user_projects_list_handler(_, request):
+  if not request.headers.get('Authorization') == 'Bearer foobar':
+    return {'status_code': 401}
+
+  return {
+    'status_code': 200,
+    'headers': {
+      'Content-Type': 'application/json',
+    },
+    'content': json.dumps([
+      {
+        "id": 2,
+        "name": "Another project",
+        "description": None,
+        "default_branch": "master",
+        "visibility": "public",
+        "path": "anotherproject",
+        "path_with_namespace": "knownuser/anotherproject",
+        "last_activity_at": "2013-09-30T13:46:02Z",
+        "web_url": "http://example.com/knownuser/anotherproject",
+      }
+    ]),
+  }
+
+
+@contextmanager
+def get_gitlab_trigger(dockerfile_path='', add_permissions=True, missing_avatar_url=False):
+  handlers = [user_handler, users_handler, project_branches_handler, project_tree_handler,
+              project_handler, get_projects_handler(add_permissions), tag_handler,
+              project_branch_handler, get_group_handler(missing_avatar_url), dockerfile_handler,
+              sub_dockerfile_handler, namespace_handler, user_namespace_handler, namespaces_handler,
+              commit_handler, create_deploykey_handler, delete_deploykey_handker,
+              create_hook_handler, delete_hook_handler, project_tags_handler, 
+              user_projects_list_handler, catchall_handler]
+
+  with HTTMock(*handlers):
     trigger_obj = AttrDict(dict(auth_token='foobar', id='sometrigger'))
     trigger = GitLabBuildTrigger(trigger_obj, {
       'build_source': 'foo/bar',
@@ -12,208 +590,8 @@ def get_gitlab_trigger(dockerfile_path=''):
       'username': 'knownuser'
     })
 
-  trigger._get_authorized_client = get_mock_gitlab(with_nulls=False)
+    client = gitlab.Gitlab('http://fakegitlab', oauth_token='foobar', timeout=20, api_version=4)
-  return trigger
+    client.auth()
 
-def adddeploykey_mock(project_id, name, public_key):
+    trigger._get_authorized_client = lambda: client
-  return {'id': 'foo'}
+    yield trigger
-
-def addprojecthook_mock(project_id, webhook_url, push=False):
-  return {'id': 'foo'}
-
-def get_currentuser_mock():
-  return {
-    'username': 'knownuser'
-  }
-
-def project(namespace, name, is_org=False):
-  project_access = None
-
-  if name != 'null':
-    if namespace == 'knownuser':
-      project_access = {
-        'access_level': 50,
-      }
-    else:
-      project_access = {
-        'access_level': 0,
-      }
-
-  data = {
-    'id': '%s/%s' % (namespace, name),
-    'default_branch': 'master',
-    'namespace': {
-      'id': namespace,
-      'path': namespace,
-      'name': namespace,
-    },
-    'path': name,
-    'path_with_namespace': '%s/%s' % (namespace, name),
-    'description': 'some %s repo' % name,
-    'last_activity_at': str(datetime.utcfromtimestamp(0)),
-    'web_url': 'https://bitbucket.org/%s/%s' % (namespace, name),
-    'ssh_url_to_repo': 'git://%s/%s' % (namespace, name),
-    'public': name != 'somerepo',
-    'permissions': {
-      'project_access': project_access,
-      'group_access': {'access_level': 0},
-    },
-    'owner': {
-      'avatar_url': 'avatarurl',
-    }
-  }
-
-  if name == 'null':
-    del data['owner']['avatar_url']
-    data['namespace']['avatar'] = None
-  elif is_org:
-    del data['owner']['avatar_url']
-    data['namespace']['avatar'] = {'url': 'avatarurl'}
-
-  return data
-
-def getprojects_mock(with_nulls=False):
-  if with_nulls:
-    def _getprojs(page=1, per_page=100):
-      return [
-        project('someorg', 'null', is_org=True),
-      ]
-    return _getprojs
-
-  else:
-    def _getprojs(page=1, per_page=100):
-      return [
-        project('knownuser', 'somerepo'),
-        project('someorg', 'somerepo', is_org=True),
-        project('someorg', 'anotherrepo', is_org=True),
-      ]
-    return _getprojs
-
-def getproject_mock(project_name):
-  if project_name == 'knownuser/somerepo':
-    return project('knownuser', 'somerepo')
-
-  if project_name == 'foo/bar':
-    return project('foo', 'bar', is_org=True)
-
-  return False
-
-
-def getbranches_mock(project_id):
-  return [
-    {
-      'name': 'master',
-      'commit': {
-        'id': 'aaaaaaa',
-      }
-    },
-    {
-      'name': 'otherbranch',
-      'commit': {
-        'id': 'aaaaaaa',
-      }
-    },
-  ]
-
-def getrepositorytags_mock(project_id):
-  return [
-    {
-      'name': 'sometag',
-      'commit': {
-        'id': 'aaaaaaa',
-      }
-    },
-    {
-      'name': 'someothertag',
-      'commit': {
-        'id': 'aaaaaaa',
-      }
-    },
-  ]
-
-def getrepositorytree_mock(project_id, ref_name='master'):
-  return [
-    {'name': 'README'},
-    {'name': 'Dockerfile'},
-  ]
-
-def getrepositorycommit_mock(project_id, commit_sha):
-  if commit_sha != 'aaaaaaa':
-    return False
-
-  return {
-    'id': 'aaaaaaa',
-    'message': 'some message',
-    'committed_date': 'now',
-  }
-
-def getusers_mock(search=None):
-  if search == 'knownuser':
-    return [
-      {
-        'username':  'knownuser',
-        'avatar_url': 'avatarurl',
-      }
-    ]
-
-  return False
-
-def getbranch_mock(repo_id, branch):
-  if branch != 'master' and branch != 'otherbranch':
-    return False
-
-  return {
-    'name': branch,
-    'commit': {
-      'id': 'aaaaaaa',
-    }
-  }
-
-def gettag_mock(repo_id, tag):
-  if tag != 'sometag' and tag != 'someothertag':
-    return False
-
-  return {
-    'name': tag,
-    'commit': {
-      'id': 'aaaaaaa',
-    }
-  }
-
-def getrawfile_mock(repo_id, branch_name, path):
-  if path == 'Dockerfile':
-    return 'hello world'
-
-  if path == 'somesubdir/Dockerfile':
-    return 'hi universe'
-
-  return False
-
-def get_mock_gitlab(with_nulls=False):
-  def _get_mock():
-    mock_gitlab = Mock()
-    mock_gitlab.host = 'https://bitbucket.org'
-
-    mock_gitlab.currentuser = Mock(side_effect=get_currentuser_mock)
-    mock_gitlab.getusers = Mock(side_effect=getusers_mock)
-
-    mock_gitlab.getprojects = Mock(side_effect=getprojects_mock(with_nulls))
-    mock_gitlab.getproject = Mock(side_effect=getproject_mock)
-    mock_gitlab.getbranches = Mock(side_effect=getbranches_mock)
-
-    mock_gitlab.getbranch = Mock(side_effect=getbranch_mock)
-    mock_gitlab.gettag = Mock(side_effect=gettag_mock)
-
-    mock_gitlab.getrepositorytags = Mock(side_effect=getrepositorytags_mock)
-    mock_gitlab.getrepositorytree = Mock(side_effect=getrepositorytree_mock)
-    mock_gitlab.getrepositorycommit = Mock(side_effect=getrepositorycommit_mock)
-
-    mock_gitlab.getrawfile = Mock(side_effect=getrawfile_mock)
-
-    mock_gitlab.adddeploykey = Mock(side_effect=adddeploykey_mock)
-    mock_gitlab.addprojecthook = Mock(side_effect=addprojecthook_mock)
-    mock_gitlab.deletedeploykey = Mock(return_value=True)
-    mock_gitlab.deleteprojecthook = Mock(return_value=True)
-    return mock_gitlab
-
-  return _get_mock

buildtrigger/test/test_githosthandler.py

@@ -3,12 +3,11 @@ import pytest
 from buildtrigger.triggerutil import TriggerStartException
 from buildtrigger.test.bitbucketmock import get_bitbucket_trigger
 from buildtrigger.test.githubmock import get_github_trigger
-from buildtrigger.test.gitlabmock import get_gitlab_trigger
 from endpoints.building import PreparedBuild
 
 # Note: This test suite executes a common set of tests against all the trigger types specified
 # in this fixture. Each trigger's mock is expected to return the same data for all of these calls.
-@pytest.fixture(params=[get_github_trigger(), get_bitbucket_trigger(), get_gitlab_trigger()])
+@pytest.fixture(params=[get_github_trigger(), get_bitbucket_trigger()])
 def githost_trigger(request):
   return request.param
 

buildtrigger/test/test_gitlabhandler.py

@@ -3,19 +3,20 @@ import pytest
 
 from mock import Mock
 
-from buildtrigger.test.gitlabmock import get_gitlab_trigger, get_mock_gitlab
+from buildtrigger.test.gitlabmock import get_gitlab_trigger
 from buildtrigger.triggerutil import (SkipRequestException, ValidationRequestException,
-                                      InvalidPayloadException)
+                                      InvalidPayloadException, TriggerStartException)
 from endpoints.building import PreparedBuild
 from util.morecollections import AttrDict
 
-@pytest.fixture
+@pytest.fixture()
 def gitlab_trigger():
-  return get_gitlab_trigger()
+  with get_gitlab_trigger() as t:
+    yield t
 
 
 def test_list_build_subdirs(gitlab_trigger):
-  assert gitlab_trigger.list_build_subdirs() == ['/Dockerfile']
+  assert gitlab_trigger.list_build_subdirs() == ['Dockerfile']
 
 
 @pytest.mark.parametrize('dockerfile_path, contents', [
@@ -24,7 +25,7 @@ def test_list_build_subdirs(gitlab_trigger):
   ('unknownpath', None),
 ])
 def test_load_dockerfile_contents(dockerfile_path, contents):
-  trigger = get_gitlab_trigger(dockerfile_path)
+  with get_gitlab_trigger(dockerfile_path=dockerfile_path) as trigger:
     assert trigger.load_dockerfile_contents() == contents
 
 
@@ -37,23 +38,47 @@ def test_lookup_user(email, expected_response, gitlab_trigger):
   assert gitlab_trigger.lookup_user(email) == expected_response
 
 
-def test_null_permissions(gitlab_trigger):
+def test_null_permissions():
-  gitlab_trigger._get_authorized_client = get_mock_gitlab(with_nulls=True)
+  with get_gitlab_trigger(add_permissions=False) as trigger:
-  sources = gitlab_trigger.list_build_sources_for_namespace('someorg')
+    sources = trigger.list_build_sources_for_namespace('someorg')
     source = sources[0]
     assert source['has_admin_permissions']
 
 
-def test_null_avatar(gitlab_trigger):
+def test_list_build_sources():
-  gitlab_trigger._get_authorized_client = get_mock_gitlab(with_nulls=True)
+  with get_gitlab_trigger() as trigger:
-  namespace_data = gitlab_trigger.list_build_source_namespaces()
+    sources = trigger.list_build_sources_for_namespace('someorg')
+    assert sources == [
+      {
+        'last_updated': 1380548762,
+        'name': u'someproject',
+        'url': u'http://example.com/someorg/someproject',
+        'private': True,
+        'full_name': u'someorg/someproject',
+        'has_admin_permissions': False,
+        'description': ''
+      },
+      {
+        'last_updated': 1380548762,
+        'name': u'anotherproject',
+        'url': u'http://example.com/someorg/anotherproject',
+        'private': False,
+        'full_name': u'someorg/anotherproject',
+        'has_admin_permissions': True,
+        'description': '',
+      }]
+
+
+def test_null_avatar():
+  with get_gitlab_trigger(missing_avatar_url=True) as trigger:
+    namespace_data = trigger.list_build_source_namespaces()
     expected = {
       'avatar_url': None,
       'personal': False,
-    'title': 'someorg',
+      'title': u'someorg',
-    'url': 'https://bitbucket.org/someorg',
+      'url': u'http://gitlab.com/groups/someorg',
       'score': 1,
-    'id': 'someorg',
+      'id': '2',
     }
 
     assert namespace_data == [expected]
@@ -112,3 +137,95 @@ def test_handle_trigger_request(gitlab_trigger, payload, expected_error, expecte
     assert isinstance(gitlab_trigger.handle_trigger_request(request), PreparedBuild)
 
 
+@pytest.mark.parametrize('run_parameters, expected_error, expected_message', [
+  # No branch or tag specified: use the commit of the default branch.
+  ({}, None, None),
+
+  # Invalid branch.
+  ({'refs': {'kind': 'branch', 'name': 'invalid'}}, TriggerStartException,
+   'Could not find branch in repository'),
+
+  # Invalid tag.
+  ({'refs': {'kind': 'tag', 'name': 'invalid'}}, TriggerStartException,
+   'Could not find tag in repository'),
+
+  # Valid branch.
+  ({'refs': {'kind': 'branch', 'name': 'master'}}, None, None),
+
+  # Valid tag.
+  ({'refs': {'kind': 'tag', 'name': 'sometag'}}, None, None),
+])
+def test_manual_start(run_parameters, expected_error, expected_message, gitlab_trigger):
+  if expected_error is not None:
+    with pytest.raises(expected_error) as ipe:
+      gitlab_trigger.manual_start(run_parameters)
+    assert ipe.value.message == expected_message
+  else:
+    assert isinstance(gitlab_trigger.manual_start(run_parameters), PreparedBuild)
+
+
+def test_activate_and_deactivate(gitlab_trigger):
+  _, private_key = gitlab_trigger.activate('http://some/url')
+  assert 'private_key' in private_key
+
+  gitlab_trigger.deactivate()
+
+
+@pytest.mark.parametrize('name, expected', [
+  ('refs', [
+    {'kind': 'branch', 'name': 'master'},
+    {'kind': 'branch', 'name': 'otherbranch'},
+    {'kind': 'tag', 'name': 'sometag'},
+    {'kind': 'tag', 'name': 'someothertag'},
+  ]),
+  ('tag_name', set(['sometag', 'someothertag'])),
+  ('branch_name', set(['master', 'otherbranch'])),
+  ('invalid', None)
+])
+def test_list_field_values(name, expected, gitlab_trigger):
+  if expected is None:
+    assert gitlab_trigger.list_field_values(name) is None
+  elif isinstance(expected, set):
+    assert set(gitlab_trigger.list_field_values(name)) == set(expected)
+  else:
+    assert gitlab_trigger.list_field_values(name) == expected
+
+
+@pytest.mark.parametrize('namespace, expected', [
+  ('', []),
+  ('unknown', []),
+
+  ('knownuser', [
+    {
+      'last_updated': 1380548762,
+      'name': u'anotherproject',
+      'url': u'http://example.com/knownuser/anotherproject',
+      'private': False,
+      'full_name': u'knownuser/anotherproject',
+      'has_admin_permissions': True,
+      'description': ''
+    },
+  ]),
+
+  ('someorg', [
+    {
+      'last_updated': 1380548762,
+      'name': u'someproject',
+      'url': u'http://example.com/someorg/someproject',
+      'private': True,
+      'full_name': u'someorg/someproject',
+      'has_admin_permissions': False,
+      'description': ''
+    },
+    {
+      'last_updated': 1380548762,
+      'name': u'anotherproject',
+      'url': u'http://example.com/someorg/anotherproject',
+      'private': False,
+      'full_name': u'someorg/anotherproject',
+      'has_admin_permissions': True,
+      'description': '',
+    }]),
+])
+def test_list_build_sources_for_namespace(namespace, expected, gitlab_trigger):
+  assert gitlab_trigger.list_build_sources_for_namespace(namespace) == expected

buildtrigger/triggerutil.py

@@ -3,37 +3,43 @@ import io
 import logging
 import re
 
-class InvalidPayloadException(Exception):
+class TriggerException(Exception):
   pass
 
-class BuildArchiveException(Exception):
+class TriggerAuthException(TriggerException):
   pass
 
-class InvalidServiceException(Exception):
+class InvalidPayloadException(TriggerException):
   pass
 
-class TriggerActivationException(Exception):
+class BuildArchiveException(TriggerException):
   pass
 
-class TriggerDeactivationException(Exception):
+class InvalidServiceException(TriggerException):
   pass
 
-class TriggerStartException(Exception):
+class TriggerActivationException(TriggerException):
   pass
 
-class ValidationRequestException(Exception):
+class TriggerDeactivationException(TriggerException):
   pass
 
-class SkipRequestException(Exception):
+class TriggerStartException(TriggerException):
   pass
 
-class EmptyRepositoryException(Exception):
+class ValidationRequestException(TriggerException):
   pass
 
-class RepositoryReadException(Exception):
+class SkipRequestException(TriggerException):
   pass
 
-class TriggerProviderException(Exception):
+class EmptyRepositoryException(TriggerException):
+  pass
+
+class RepositoryReadException(TriggerException):
+  pass
+
+class TriggerProviderException(TriggerException):
   pass
 
 logger = logging.getLogger(__name__)

endpoints/api/trigger.py

@@ -11,9 +11,7 @@ from app import app
 from auth.permissions import (UserAdminPermission, AdministerOrganizationPermission,
                               ReadRepositoryPermission, AdministerRepositoryPermission)
 from buildtrigger.basehandler import BuildTriggerHandler
-from buildtrigger.triggerutil import (TriggerDeactivationException,
+from buildtrigger.triggerutil import TriggerException, EmptyRepositoryException
-                                      TriggerActivationException, EmptyRepositoryException,
-                                      RepositoryReadException, TriggerStartException)
 from data import model
 from data.model.build import update_build_trigger
 from endpoints.api import (RepositoryParamResource, nickname, resource, require_repo_admin,
@@ -118,7 +116,7 @@ class BuildTrigger(RepositoryParamResource):
     if handler.is_active():
       try:
         handler.deactivate()
-      except TriggerDeactivationException as ex:
+      except TriggerException as ex:
         # We are just going to eat this error
         logger.warning('Trigger deactivation problem: %s', ex)
 
@@ -178,7 +176,7 @@ class BuildTriggerSubdirs(RepositoryParamResource):
           'contextMap': {},
           'dockerfile_paths': [],
         }
-      except RepositoryReadException as exc:
+      except TriggerException as exc:
         return {
           'status': 'error',
           'message': exc.message,
@@ -264,7 +262,7 @@ class BuildTriggerActivate(RepositoryParamResource):
         if 'private_key' in private_config:
           trigger.private_key = private_config['private_key']
 
-      except TriggerActivationException as exc:
+      except TriggerException as exc:
         write_token.delete_instance()
         raise request_error(message=exc.message)
 
@@ -332,7 +330,7 @@ class BuildTriggerAnalyze(RepositoryParamResource):
                                          new_config_dict,
                                          AdministerOrganizationPermission(namespace_name).can())
       return trigger_analyzer.analyze_trigger()
-    except RepositoryReadException as rre:
+    except TriggerException as rre:
       return {
         'status': 'error',
         'message': 'Could not analyze the repository: %s' % rre.message,
@@ -391,7 +389,7 @@ class ActivateBuildTrigger(RepositoryParamResource):
       run_parameters = request.get_json()
       prepared = handler.manual_start(run_parameters=run_parameters)
       build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
-    except TriggerStartException as tse:
+    except TriggerException as tse:
       raise InvalidRequest(tse.message)
     except MaximumBuildsQueuedException:
       abort(429, message='Maximum queued build rate exceeded.')
@@ -494,7 +492,7 @@ class BuildTriggerSources(RepositoryParamResource):
         return {
           'sources': handler.list_build_sources_for_namespace(namespace)
         }
-      except RepositoryReadException as rre:
+      except TriggerException as rre:
         raise InvalidRequest(rre.message)
     else:
       raise Unauthorized()
@@ -522,7 +520,7 @@ class BuildTriggerSourceNamespaces(RepositoryParamResource):
         return {
           'namespaces': handler.list_build_source_namespaces()
         }
-      except RepositoryReadException as rre:
+      except TriggerException as rre:
         raise InvalidRequest(rre.message)
     else:
       raise Unauthorized()

requirements-nover.txt

@@ -5,7 +5,6 @@
 -e git+https://github.com/NateFerrero/oauth2lib.git#egg=oauth2lib
 -e git+https://github.com/coreos/mockldap.git@v0.1.x#egg=mockldap
 -e git+https://github.com/coreos/py-bitbucket.git#egg=py-bitbucket
--e git+https://github.com/coreos/pyapi-gitlab.git@timeout#egg=pyapi-gitlab
 -e git+https://github.com/coreos/resumablehashlib.git#egg=resumablehashlib
 -e git+https://github.com/jepcastelein/marketo-rest-python.git#egg=marketorestpython
 -e git+https://github.com/app-registry/appr-server.git@c2ef3b88afe926a92ef5f2e11e7d4a259e286a17#egg=cnr_server  # naming has changed
@@ -58,6 +57,7 @@ pyjwkest
 pyjwt
 pymysql==0.6.7  # Remove version when baseimage has Python 2.7.9+
 python-dateutil
+python-gitlab
 python-keystoneclient
 python-ldap
 python-magic

requirements.txt

@@ -2,7 +2,6 @@ aiowsgi==0.6
 alembic==0.9.8
 -e git+https://github.com/coreos/mockldap.git@59a46efbe8c7cd8146a87a7c4f2b09746b953e11#egg=mockldap
 -e git+https://github.com/coreos/py-bitbucket.git@55a1ada645f2fb6369147996ec71edd7828d91c8#egg=py_bitbucket
--e git+https://github.com/coreos/pyapi-gitlab.git@136c3970d591136a4f766a846c5d22aad52e124f#egg=pyapi_gitlab
 -e git+https://github.com/coreos/resumablehashlib.git@b1b631249589b07adf40e0ee545b323a501340b4#egg=resumablehashlib
 -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601
 -e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode
@@ -113,6 +112,7 @@ pyparsing==2.2.0
 PyPDF2==1.26.0
 python-dateutil==2.6.1
 python-editor==1.0.3
+python-gitlab==1.4.0
 python-keystoneclient==3.15.0
 python-ldap==2.5.2
 python-magic==0.4.15

static/js/directives/ui/manage-trigger/manage-trigger.component.html

@@ -35,7 +35,7 @@
               <img class="namespace-avatar" ng-src="{{ ::item.avatar_url }}" ng-if="::item.avatar_url">
               <span class="anchor"
                     href="{{ ::item.url }}"
-                    is-text-only="::!item.url">{{ ::item.id }}</span>
+                    is-text-only="::!item.url">{{ ::item.title }}</span>
             </script>
           </cor-table-col>
           <cor-table-col title="Importance"