quay/endpoints/trigger.py

import logging
import io
import os.path
import tarfile
import base64
import re

from github import Github, UnknownObjectException, GithubException
from tempfile import SpooledTemporaryFile

from app import app, userfiles as user_files, github_trigger
from util.tarfileappender import TarfileAppender


client = app.config['HTTPCLIENT']


logger = logging.getLogger(__name__)


TARBALL_MIME = 'application/gzip'
CHUNK_SIZE = 512 * 1024


def should_skip_commit(message):
  return '[skip build]' in message or '[build skip]' in message


class BuildArchiveException(Exception):
  pass

class InvalidServiceException(Exception):
  pass

class TriggerActivationException(Exception):
  pass

class TriggerDeactivationException(Exception):
  pass

class TriggerStartException(Exception):
  pass

class ValidationRequestException(Exception):
  pass

class SkipRequestException(Exception):
  pass

class EmptyRepositoryException(Exception):
  pass

class RepositoryReadException(Exception):
  pass


class BuildTrigger(object):
  def __init__(self):
    pass

  def dockerfile_url(self, auth_token, config):
    """
    Returns the URL at which the Dockerfile for the trigger can be found or None if none/not applicable.
    """
    return None

  def load_dockerfile_contents(self, auth_token, config):
    """
    Loads the Dockerfile found for the trigger's config and returns them or None if none could
    be found/loaded.
    """
    return None

  def list_build_sources(self, auth_token):
    """
    Take the auth information for the specific trigger type and load the
    list of build sources(repositories).
    """
    raise NotImplementedError

  def list_build_subdirs(self, auth_token, config):
    """
    Take the auth information and the specified config so far and list all of
    the possible subdirs containing dockerfiles.
    """
    raise NotImplementedError

  def handle_trigger_request(self, request, auth_token, config):
    """
    Transform the incoming request data into a set of actions. Returns a tuple
    of usefiles resource id, docker tags, build name, and resource subdir.
    """
    raise NotImplementedError

  def is_active(self, config):
    """
    Returns True if the current build trigger is active. Inactive means further
    setup is needed.
    """
    raise NotImplementedError

  def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
    """
    Activates the trigger for the service, with the given new configuration.
    Returns new configuration that should be stored if successful.
    """
    raise NotImplementedError

  def deactivate(self, auth_token, config):
    """
    Deactivates the trigger for the service, removing any hooks installed in
    the remote service. Returns the new config that should be stored if this
    trigger is going to be re-activated.
    """
    raise NotImplementedError

  def manual_start(self, auth_token, config, run_parameters = None):
    """
    Manually creates a repository build for this trigger.
    """
    raise NotImplementedError

  def list_field_values(self, auth_token, config, field_name):
    """
    Lists all values for the given custom trigger field. For example, a trigger might have a
    field named "branches", and this method would return all branches.
    """
    raise NotImplementedError

  @classmethod
  def service_name(cls):
    """
    Particular service implemented by subclasses.
    """
    raise NotImplementedError

  @classmethod
  def get_trigger_for_service(cls, service):
    for subc in cls.__subclasses__():
      if subc.service_name() == service:
        return subc()

    raise InvalidServiceException('Unable to find service: %s' % service)


def raise_unsupported():
  raise io.UnsupportedOperation


class GithubBuildTrigger(BuildTrigger):
  @staticmethod
  def _get_client(auth_token):
    return Github(auth_token,
                  base_url=github_trigger.api_endpoint(),
                  client_id=github_trigger.client_id(),
                  client_secret=github_trigger.client_secret())

  @classmethod
  def service_name(cls):
    return 'github'

  def is_active(self, config):
    return 'hook_id' in config

  def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
    new_build_source = config['build_source']
    gh_client = self._get_client(auth_token)

    try:
      to_add_webhook = gh_client.get_repo(new_build_source)
    except UnknownObjectException:
      msg = 'Unable to find GitHub repository for source: %s'
      raise TriggerActivationException(msg % new_build_source)

    webhook_config = {
      'url': standard_webhook_url,
      'content_type': 'json',
    }

    try:
      hook = to_add_webhook.create_hook('web', webhook_config)
      config['hook_id'] = hook.id
      config['master_branch'] = to_add_webhook.default_branch
    except GithubException:
      msg = 'Unable to create webhook on repository: %s'
      raise TriggerActivationException(msg % new_build_source)

    return config

  def deactivate(self, auth_token, config):
    gh_client = self._get_client(auth_token)

    try:
      repo = gh_client.get_repo(config['build_source'])
      to_delete = repo.get_hook(config['hook_id'])
      to_delete.delete()
    except GithubException:
      msg = 'Unable to remove hook: %s' % config['hook_id']
      raise TriggerDeactivationException(msg)

    config.pop('hook_id', None)

    return config

  def list_build_sources(self, auth_token):
    gh_client = self._get_client(auth_token)
    usr = gh_client.get_user()

    personal = {
      'personal': True,
      'repos': [repo.full_name for repo in usr.get_repos()],
      'info': {
        'name': usr.login,
        'avatar_url': usr.avatar_url,
      }
    }

    repos_by_org = [personal]

    for org in usr.get_orgs():
      repo_list = []
      for repo in org.get_repos(type='member'):
        repo_list.append(repo.full_name)

      repos_by_org.append({
        'personal': False,
        'repos': repo_list,
        'info': {
          'name': org.name or org.login,
          'avatar_url': org.avatar_url
        }
      })

    return repos_by_org

  def matches_ref(self, ref, regex):
    match_string = ref.split('/', 1)[1]
    if not regex:
      return False

    m = regex.match(match_string)
    if not m:
      return False

    return len(m.group(0)) == len(match_string)

  def list_build_subdirs(self, auth_token, config):
    gh_client = self._get_client(auth_token)
    source = config['build_source']

    try:
      repo = gh_client.get_repo(source)

      # Find the first matching branch.
      branches = None
      if 'branchtag_regex' in config:
        try:
          regex = re.compile(config['branchtag_regex'])
          branches = [branch.name for branch in repo.get_branches()
                                  if self.matches_ref('refs/heads/' + branch.name, regex)]
        except:
          pass

      branches = branches or [repo.default_branch or 'master']
      default_commit = repo.get_branch(branches[0]).commit
      commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)

      return [os.path.dirname(elem.path) for elem in commit_tree.tree
              if (elem.type == u'blob' and
                  os.path.basename(elem.path) == u'Dockerfile')]
    except GithubException as ge:
      message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
      if message == 'Branch not found':
        raise EmptyRepositoryException()

      raise RepositoryReadException(message)

  def dockerfile_url(self,  auth_token, config):
    source = config['build_source']
    subdirectory = config.get('subdir', '')
    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'

    gh_client = self._get_client(auth_token)
    try:
      repo = gh_client.get_repo(source)
      master_branch = repo.default_branch or 'master'
      return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
    except GithubException as ge:
      return None

  def load_dockerfile_contents(self, auth_token, config):
    gh_client = self._get_client(auth_token)

    source = config['build_source']
    subdirectory = config.get('subdir', '')
    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'

    try:
      repo = gh_client.get_repo(source)
      file_info = repo.get_file_contents(path)
      if file_info is None:
        return None

      content = file_info.content
      if file_info.encoding == 'base64':
        content = base64.b64decode(content)
      return content

    except GithubException as ge:
      message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source)
      raise RepositoryReadException(message)

  @staticmethod
  def _build_commit_info(repo, commit_sha):
    try:
      commit = repo.get_commit(commit_sha)
    except GithubException:
      logger.exception('Could not load data for commit')
      return

    commit_info = {
      'url': commit.html_url,
      'message': commit.commit.message,
      'date': commit.last_modified
    }

    if commit.author:
      commit_info['author'] = {
        'username': commit.author.login,
        'avatar_url': commit.author.avatar_url,
        'url': commit.author.html_url
      }

    if commit.committer:
      commit_info['committer'] = {
        'username': commit.committer.login,
        'avatar_url': commit.committer.avatar_url,
        'url': commit.committer.html_url
      }

    return commit_info

  @staticmethod
  def _prepare_build(config, repo, commit_sha, build_name, ref):
    # Prepare the download and upload URLs
    archive_link = repo.get_archive_link('tarball', commit_sha)
    download_archive = client.get(archive_link, stream=True)

    tarball_subdir = ''
    with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
      for chunk in download_archive.iter_content(CHUNK_SIZE):
        tarball.write(chunk)

      # Seek to position 0 to make tarfile happy
      tarball.seek(0)

      # Pull out the name of the subdir that GitHub generated
      with tarfile.open(fileobj=tarball) as archive:
        tarball_subdir = archive.getnames()[0]

      # Seek to position 0 to make tarfile happy.
      tarball.seek(0)

      entries = {
        tarball_subdir + '/.git/HEAD': commit_sha,
        tarball_subdir + '/.git/objects/': None,
        tarball_subdir + '/.git/refs/': None
      }

      appender = TarfileAppender(tarball, entries).get_stream()
      dockerfile_id = user_files.store_file(appender, TARBALL_MIME)

    logger.debug('Successfully prepared job')

    # compute the tag(s)
    branch = ref.split('/')[-1]
    tags = {branch}

    if branch == repo.default_branch:
      tags.add('latest')

    logger.debug('Pushing to tags: %s' % tags)

    # compute the subdir
    repo_subdir = config['subdir']
    joined_subdir = os.path.join(tarball_subdir, repo_subdir)
    logger.debug('Final subdir: %s' % joined_subdir)

    # compute the metadata
    metadata = {
      'commit_sha': commit_sha,
      'ref': ref,
      'default_branch': repo.default_branch,
    }

    # add the commit info.
    commit_info = GithubBuildTrigger._build_commit_info(repo, commit_sha)
    if commit_info is not None:
      metadata['commit_info'] = commit_info

    return dockerfile_id, list(tags), build_name, joined_subdir, metadata

  @staticmethod
  def get_display_name(sha):
    return sha[0:7]

  def handle_trigger_request(self, request, auth_token, config):
    payload = request.get_json()
    if not payload or payload.get('head_commit') is None:
      raise SkipRequestException()

    if 'zen' in payload:
      raise ValidationRequestException()

    logger.debug('Payload %s', payload)
    ref = payload['ref']
    commit_sha = payload['head_commit']['id']
    commit_message = payload['head_commit'].get('message', '')

    if 'branchtag_regex' in config:
      try:
        regex = re.compile(config['branchtag_regex'])
      except:
        regex = re.compile('.*')

      if not self.matches_ref(ref, regex):
        raise SkipRequestException()

    if should_skip_commit(commit_message):
      raise SkipRequestException()

    short_sha = GithubBuildTrigger.get_display_name(commit_sha)

    gh_client = self._get_client(auth_token)

    repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
                                payload['repository']['name'])
    repo = gh_client.get_repo(repo_full_name)

    logger.debug('Github repo: %s', repo)

    return GithubBuildTrigger._prepare_build(config, repo, commit_sha,
                                             short_sha, ref)

  def manual_start(self, auth_token, config, run_parameters = None):
    try:
      source = config['build_source']
      run_parameters = run_parameters or {}

      gh_client = self._get_client(auth_token)
      repo = gh_client.get_repo(source)
      branch_name = run_parameters.get('branch_name') or repo.default_branch
      branch = repo.get_branch(branch_name)
      branch_sha = branch.commit.sha
      commit_info = branch.commit
      short_sha = GithubBuildTrigger.get_display_name(branch_sha)
      ref = 'refs/heads/%s' % (branch_name)

      return self._prepare_build(config, repo, branch_sha, short_sha, ref)
    except GithubException as ghe:
      raise TriggerStartException(ghe.data['message'])


  def list_field_values(self, auth_token, config, field_name):
    if field_name == 'refs':
      branches = self.list_field_values(auth_token, config, 'branch_name')
      tags = self.list_field_values(auth_token, config, 'tag_name')

      return ([{'kind': 'branch', 'name': b} for b in branches] +
              [{'kind': 'tag', 'name': tag} for tag in tags])

    if field_name == 'tag_name':
      gh_client = self._get_client(auth_token)
      source = config['build_source']
      repo = gh_client.get_repo(source)
      return [tag.name for tag in repo.get_tags()]

    if field_name == 'branch_name':
      gh_client = self._get_client(auth_token)
      source = config['build_source']
      repo = gh_client.get_repo(source)
      branches = [branch.name for branch in repo.get_branches()]

      if not repo.default_branch in branches:
        branches.insert(0, repo.default_branch)

      if branches[0] != repo.default_branch:
        branches.remove(repo.default_branch)
        branches.insert(0, repo.default_branch)

      return branches

    return None