quay/buildtrigger/bitbuckethandler.py

import logging
import os
import re
from calendar import timegm

import dateutil.parser
from bitbucket import BitBucket
from jsonschema import validate

from app import app, get_app_url
from buildtrigger.basehandler import BuildTriggerHandler
from buildtrigger.triggerutil import (RepositoryReadException, TriggerActivationException,
                                      TriggerDeactivationException, TriggerStartException,
                                      InvalidPayloadException, TriggerProviderException,
                                      determine_build_ref, raise_if_skipped_build,
                                      find_matching_branches)
from util.dict_wrappers import JSONPathDict, SafeDictSetter
from util.security.ssh import generate_ssh_keypair

logger = logging.getLogger(__name__)

_BITBUCKET_COMMIT_URL = 'https://bitbucket.org/%s/commits/%s'
_RAW_AUTHOR_REGEX = re.compile(r'.*<(.+)>')

BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA = {
  'type': 'object',
  'properties': {
    'repository': {
      'type': 'object',
      'properties': {
        'full_name': {
          'type': 'string',
        },
      },
      'required': ['full_name'],
    }, # /Repository
    'push': {
      'type': 'object',
      'properties': {
        'changes': {
          'type': 'array',
          'items': {
            'type': 'object',
            'properties': {
              'new': {
                'type': 'object',
                'properties': {
                  'target': {
                    'type': 'object',
                    'properties': {
                      'hash': {
                        'type': 'string'
                      },
                      'message': {
                        'type': 'string'
                      },
                      'date': {
                        'type': 'string'
                      },
                      'author': {
                        'type': 'object',
                        'properties': {
                          'user': {
                            'type': 'object',
                            'properties': {
                              'username': {
                                'type': 'string',
                              },
                              'links': {
                                'type': 'object',
                                'properties': {
                                  'html': {
                                    'type': 'object',
                                    'properties': {
                                      'href': {
                                        'type': 'string',
                                      },
                                    },
                                    'required': ['href'],
                                  },
                                  'avatar': {
                                    'type': 'object',
                                    'properties': {
                                      'href': {
                                        'type': 'string',
                                      },
                                    },
                                    'required': ['href'],
                                  },
                                },
                                'required': ['html', 'avatar'],
                              }, # /User
                            },
                            'required': ['username'],
                          }, # /Author
                        },
                      },
                      'links': {
                        'type': 'object',
                        'properties': {
                          'html': {
                            'type': 'object',
                            'properties': {
                              'href': {
                                'type': 'string',
                              },
                            },
                            'required': ['href'],
                          },
                        },
                        'required': ['html'],
                      }, # /Links
                    },
                    'required': ['hash', 'message', 'date'],
                  }, # /Target
                },
                'required': ['name', 'target'],
              }, # /New
            },
          }, # /Changes item
        }, # /Changes
      },
      'required': ['changes'],
    }, # / Push
  },
  'actor': {
    'type': 'object',
    'properties': {
      'username': {
        'type': 'string',
      },
      'links': {
        'type': 'object',
        'properties': {
          'html': {
            'type': 'object',
            'properties': {
              'href': {
                'type': 'string',
              },
            },
            'required': ['href'],
          },
          'avatar': {
            'type': 'object',
            'properties': {
              'href': {
                'type': 'string',
              },
            },
            'required': ['href'],
          },
        },
        'required': ['html', 'avatar'],
      },
    },
    'required': ['username'],
  }, # /Actor
  'required': ['push', 'repository'],
} # /Root

BITBUCKET_COMMIT_INFO_SCHEMA = {
  'type': 'object',
  'properties': {
    'node': {
      'type': 'string',
    },
    'message': {
      'type': 'string',
    },
    'timestamp': {
      'type': 'string',
    },
    'raw_author': {
      'type': 'string',
    },
  },
  'required': ['node', 'message', 'timestamp']
}

def get_transformed_commit_info(bb_commit, ref, default_branch, repository_name, lookup_author):
  """ Returns the BitBucket commit information transformed into our own
      payload format.
  """
  try:
    validate(bb_commit, BITBUCKET_COMMIT_INFO_SCHEMA)
  except Exception as exc:
    logger.exception('Exception when validating Bitbucket commit information: %s from %s', exc.message, bb_commit)
    raise InvalidPayloadException(exc.message)

  commit = JSONPathDict(bb_commit)

  config = SafeDictSetter()
  config['commit'] = commit['node']
  config['ref'] = ref
  config['default_branch'] = default_branch
  config['git_url'] = 'git@bitbucket.org:%s.git' % repository_name

  config['commit_info.url'] = _BITBUCKET_COMMIT_URL % (repository_name, commit['node'])
  config['commit_info.message'] = commit['message']
  config['commit_info.date'] = commit['timestamp']

  match = _RAW_AUTHOR_REGEX.match(commit['raw_author'])
  if match:
    author = lookup_author(match.group(1))
    author_info = JSONPathDict(author) if author is not None else None
    if author_info:
      config['commit_info.author.username'] = author_info['user.username']
      config['commit_info.author.url'] = 'https://bitbucket.org/%s/' % author_info['user.username']
      config['commit_info.author.avatar_url'] = author_info['user.avatar']

  return config.dict_value()


def get_transformed_webhook_payload(bb_payload, default_branch=None):
  """ Returns the BitBucket webhook JSON payload transformed into our own payload
      format. If the bb_payload is not valid, returns None.
  """
  try:
    validate(bb_payload, BITBUCKET_WEBHOOK_PAYLOAD_SCHEMA)
  except Exception as exc:
    logger.exception('Exception when validating Bitbucket webhook payload: %s from %s', exc.message,
                     bb_payload)
    raise InvalidPayloadException(exc.message)

  payload = JSONPathDict(bb_payload)
  change = payload['push.changes[-1].new']
  if not change:
    return None

  is_branch = change['type'] == 'branch'
  ref = 'refs/heads/' + change['name'] if is_branch else 'refs/tags/' + change['name']

  repository_name = payload['repository.full_name']
  target = change['target']

  config = SafeDictSetter()
  config['commit'] = target['hash']
  config['ref'] = ref
  config['default_branch'] = default_branch
  config['git_url'] = 'git@bitbucket.org:%s.git' % repository_name

  config['commit_info.url'] = target['links.html.href'] or ''
  config['commit_info.message'] = target['message']
  config['commit_info.date'] = target['date']

  config['commit_info.author.username'] = target['author.user.username']
  config['commit_info.author.url'] = target['author.user.links.html.href']
  config['commit_info.author.avatar_url'] = target['author.user.links.avatar.href']

  config['commit_info.committer.username'] = payload['actor.username']
  config['commit_info.committer.url'] = payload['actor.links.html.href']
  config['commit_info.committer.avatar_url'] = payload['actor.links.avatar.href']
  return config.dict_value()


class BitbucketBuildTrigger(BuildTriggerHandler):
  """
  BuildTrigger for Bitbucket.
  """
  @classmethod
  def service_name(cls):
    return 'bitbucket'

  def _get_client(self):
    """ Returns a BitBucket API client for this trigger's config. """
    key = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_KEY', '')
    secret = app.config.get('BITBUCKET_TRIGGER_CONFIG', {}).get('CONSUMER_SECRET', '')

    trigger_uuid = self.trigger.uuid
    callback_url = '%s/oauth1/bitbucket/callback/trigger/%s' % (get_app_url(), trigger_uuid)

    return BitBucket(key, secret, callback_url, timeout=15)

  def _get_authorized_client(self):
    """ Returns an authorized API client. """
    base_client = self._get_client()
    auth_token = self.auth_token or 'invalid:invalid'
    token_parts = auth_token.split(':')
    if len(token_parts) != 2:
      token_parts = ['invalid', 'invalid']

    (access_token, access_token_secret) = token_parts
    return base_client.get_authorized_client(access_token, access_token_secret)

  def _get_repository_client(self):
    """ Returns an API client for working with this config's BB repository. """
    source = self.config['build_source']
    (namespace, name) = source.split('/')
    bitbucket_client = self._get_authorized_client()
    return bitbucket_client.for_namespace(namespace).repositories().get(name)

  def _get_default_branch(self, repository, default_value='master'):
    """ Returns the default branch for the repository or the value given. """
    (result, data, _) = repository.get_main_branch()
    if result:
      return data['name']

    return default_value

  def get_oauth_url(self):
    """ Returns the OAuth URL to authorize Bitbucket. """
    bitbucket_client = self._get_client()
    (result, data, err_msg) = bitbucket_client.get_authorization_url()
    if not result:
      raise TriggerProviderException(err_msg)

    return data

  def exchange_verifier(self, verifier):
    """ Exchanges the given verifier token to setup this trigger. """
    bitbucket_client = self._get_client()
    access_token = self.config.get('access_token', '')
    access_token_secret = self.auth_token

    # Exchange the verifier for a new access token.
    (result, data, _) = bitbucket_client.verify_token(access_token, access_token_secret, verifier)
    if not result:
      return False

    # Save the updated access token and secret.
    self.set_auth_token(data[0] + ':' + data[1])

    # Retrieve the current authorized user's information and store the username in the config.
    authorized_client = self._get_authorized_client()
    (result, data, _) = authorized_client.get_current_user()
    if not result:
      return False

    username = data['user']['username']
    self.put_config_key('username', username)
    return True

  def is_active(self):
    return 'webhook_id' in self.config

  def activate(self, standard_webhook_url):
    config = self.config

    # Add a deploy key to the repository.
    public_key, private_key = generate_ssh_keypair()
    config['credentials'] = [
      {
        'name': 'SSH Public Key',
        'value': public_key,
      },
    ]

    repository = self._get_repository_client()
    (result, created_deploykey, err_msg) = repository.deploykeys().create(
      app.config['REGISTRY_TITLE'] + ' webhook key', public_key)

    if not result:
      msg = 'Unable to add deploy key to repository: %s' % err_msg
      raise TriggerActivationException(msg)

    config['deploy_key_id'] = created_deploykey['pk']

    # Add a webhook callback.
    description = 'Webhook for invoking builds on %s' % app.config['REGISTRY_TITLE_SHORT']
    webhook_events = ['repo:push']
    (result, created_webhook, err_msg) = repository.webhooks().create(
      description, standard_webhook_url, webhook_events)

    if not result:
      msg = 'Unable to add webhook to repository: %s' % err_msg
      raise TriggerActivationException(msg)

    config['webhook_id'] = created_webhook['uuid']
    self.config = config
    return config, {'private_key': private_key}

  def deactivate(self):
    config = self.config

    webhook_id = config.pop('webhook_id', None)
    deploy_key_id = config.pop('deploy_key_id', None)
    repository = self._get_repository_client()

    # Remove the webhook.
    if webhook_id is not None:
      (result, _, err_msg) = repository.webhooks().delete(webhook_id)
      if not result:
        msg = 'Unable to remove webhook from repository: %s' % err_msg
        raise TriggerDeactivationException(msg)

    # Remove the public key.
    if deploy_key_id is not None:
      (result, _, err_msg) = repository.deploykeys().delete(deploy_key_id)
      if not result:
        msg = 'Unable to remove deploy key from repository: %s' % err_msg
        raise TriggerDeactivationException(msg)

    return config

  def list_build_source_namespaces(self):
    bitbucket_client = self._get_authorized_client()
    (result, data, err_msg) = bitbucket_client.get_visible_repositories()
    if not result:
      raise RepositoryReadException('Could not read repository list: ' + err_msg)

    namespaces = {}
    for repo in data:
      owner = repo['owner']
      if owner in namespaces:
        namespaces[owner]['score'] = namespaces[owner]['score'] + 1
      else:
        namespaces[owner] = {
          'personal': owner == self.config.get('username'),
          'id': owner,
          'title': owner,
          'avatar_url': repo['logo'],
          'url': 'https://bitbucket.org/%s' % (owner),
          'score': 1,
        }

    return BuildTriggerHandler.build_namespaces_response(namespaces)

  def list_build_sources_for_namespace(self, namespace):
    def repo_view(repo):
      last_modified = dateutil.parser.parse(repo['utc_last_updated'])

      return {
        'name': repo['slug'],
        'full_name': '%s/%s' % (repo['owner'], repo['slug']),
        'description': repo['description'] or '',
        'last_updated': timegm(last_modified.utctimetuple()),
        'url': 'https://bitbucket.org/%s/%s' % (repo['owner'], repo['slug']),
        'has_admin_permissions': repo['read_only'] is False,
        'private': repo['is_private'],
      }

    bitbucket_client = self._get_authorized_client()
    (result, data, err_msg) = bitbucket_client.get_visible_repositories()
    if not result:
      raise RepositoryReadException('Could not read repository list: ' + err_msg)

    repos = [repo_view(repo) for repo in data if repo['owner'] == namespace]
    return BuildTriggerHandler.build_sources_response(repos)

  def list_build_subdirs(self):
    config = self.config
    repository = self._get_repository_client()

    # Find the first matching branch.
    repo_branches = self.list_field_values('branch_name') or []
    branches = find_matching_branches(config, repo_branches)
    if not branches:
      branches = [self._get_default_branch(repository)]

    (result, data, err_msg) = repository.get_path_contents('', revision=branches[0])
    if not result:
      raise RepositoryReadException(err_msg)

    files = set([f['path'] for f in data['files']])
    return ["/" + file_path for file_path in files if self.filename_is_dockerfile(os.path.basename(file_path))]

  def load_dockerfile_contents(self):
    repository = self._get_repository_client()
    path = self.get_dockerfile_path()

    (result, data, err_msg) = repository.get_raw_path_contents(path, revision='master')
    if not result:
      return None

    return data

  def list_field_values(self, field_name, limit=None):
    if 'build_source' not in self.config:
      return None

    source = self.config['build_source']
    (namespace, name) = source.split('/')

    bitbucket_client = self._get_authorized_client()
    repository = bitbucket_client.for_namespace(namespace).repositories().get(name)

    if field_name == 'refs':
      (result, data, _) = repository.get_branches_and_tags()
      if not result:
        return None

      branches = [b['name'] for b in data['branches']]
      tags = [t['name'] for t in data['tags']]

      return ([{'kind': 'branch', 'name': b} for b in branches] +
              [{'kind': 'tag', 'name': tag} for tag in tags])

    if field_name == 'tag_name':
      (result, data, _) = repository.get_tags()
      if not result:
        return None

      tags = list(data.keys())
      if limit:
        tags = tags[0:limit]

      return tags

    if field_name == 'branch_name':
      (result, data, _) = repository.get_branches()
      if not result:
        return None

      branches = list(data.keys())
      if limit:
        branches = branches[0:limit]

      return branches

    return None

  def get_repository_url(self):
    source = self.config['build_source']
    (namespace, name) = source.split('/')
    return 'https://bitbucket.org/%s/%s' % (namespace, name)

  def handle_trigger_request(self, request):
    payload = request.get_json()
    if payload is None:
      raise InvalidPayloadException('Missing payload')

    logger.debug('Got BitBucket request: %s', payload)

    repository = self._get_repository_client()
    default_branch = self._get_default_branch(repository)

    metadata = get_transformed_webhook_payload(payload, default_branch=default_branch)
    prepared = self.prepare_build(metadata)

    # Check if we should skip this build.
    raise_if_skipped_build(prepared, self.config)
    return prepared

  def manual_start(self, run_parameters=None):
    run_parameters = run_parameters or {}
    repository = self._get_repository_client()
    bitbucket_client = self._get_authorized_client()

    def get_branch_sha(branch_name):
      # Lookup the commit SHA for the branch.
      (result, data, _) = repository.get_branch(branch_name)
      if not result:
        raise TriggerStartException('Could not find branch in repository')

      return data['target']['hash']

    def get_tag_sha(tag_name):
      # Lookup the commit SHA for the tag.
      (result, data, _) = repository.get_tag(tag_name)
      if not result:
        raise TriggerStartException('Could not find tag in repository')

      return data['target']['hash']

    def lookup_author(email_address):
      (result, data, _) = bitbucket_client.accounts().get_profile(email_address)
      return data if result else None

    # Find the branch or tag to build.
    default_branch = self._get_default_branch(repository)
    (commit_sha, ref) = determine_build_ref(run_parameters, get_branch_sha, get_tag_sha,
                                            default_branch)

    # Lookup the commit SHA in BitBucket.
    (result, commit_info, _) = repository.changesets().get(commit_sha)
    if not result:
      raise TriggerStartException('Could not lookup commit SHA')

    # Return a prepared build for the commit.
    repository_name = '%s/%s' % (repository.namespace, repository.repository_name)
    metadata = get_transformed_commit_info(commit_info, ref, default_branch,
                                           repository_name, lookup_author)

    return self.prepare_build(metadata, is_manual=True)