quay/endpoints/trigger.py

import logging
import io
import os.path
import tarfile
import base64
import re
import json

from github import Github, UnknownObjectException, GithubException
from tempfile import SpooledTemporaryFile
from jsonschema import validate

from app import app, userfiles as user_files, github_trigger
from util.tarfileappender import TarfileAppender
from util.ssh import generate_ssh_keypair


client = app.config['HTTPCLIENT']


logger = logging.getLogger(__name__)


TARBALL_MIME = 'application/gzip'
CHUNK_SIZE = 512 * 1024


def should_skip_commit(message):
  return '[skip build]' in message or '[build skip]' in message

class InvalidPayloadException(Exception):
  pass

class BuildArchiveException(Exception):
  pass

class InvalidServiceException(Exception):
  pass

class TriggerActivationException(Exception):
  pass

class TriggerDeactivationException(Exception):
  pass

class TriggerStartException(Exception):
  pass

class ValidationRequestException(Exception):
  pass

class SkipRequestException(Exception):
  pass

class EmptyRepositoryException(Exception):
  pass

class RepositoryReadException(Exception):
  pass


class BuildTrigger(object):
  def __init__(self):
    pass

  def dockerfile_url(self, auth_token, config):
    """
    Returns the URL at which the Dockerfile for the trigger is found or None if none/not applicable.
    """
    raise NotImplementedError

  def load_dockerfile_contents(self, auth_token, config):
    """
    Loads the Dockerfile found for the trigger's config and returns them or None if none could
    be found/loaded.
    """
    raise NotImplementedError

  def list_build_sources(self, auth_token):
    """
    Take the auth information for the specific trigger type and load the
    list of build sources(repositories).
    """
    raise NotImplementedError

  def list_build_subdirs(self, auth_token, config):
    """
    Take the auth information and the specified config so far and list all of
    the possible subdirs containing dockerfiles.
    """
    raise NotImplementedError

  def handle_trigger_request(self, request, trigger):
    """
    Transform the incoming request data into a set of actions. Returns a tuple
    of usefiles resource id, docker tags, build name, and resource subdir.
    """
    raise NotImplementedError

  def is_active(self, config):
    """
    Returns True if the current build trigger is active. Inactive means further
    setup is needed.
    """
    raise NotImplementedError

  def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
    """
    Activates the trigger for the service, with the given new configuration.
    Returns new configuration that should be stored if successful.
    """
    raise NotImplementedError

  def deactivate(self, auth_token, config):
    """
    Deactivates the trigger for the service, removing any hooks installed in
    the remote service. Returns the new config that should be stored if this
    trigger is going to be re-activated.
    """
    raise NotImplementedError

  def manual_start(self, trigger, run_parameters=None):
    """
    Manually creates a repository build for this trigger.
    """
    raise NotImplementedError

  def list_field_values(self, auth_token, config, field_name):
    """
    Lists all values for the given custom trigger field. For example, a trigger might have a
    field named "branches", and this method would return all branches.
    """
    raise NotImplementedError

  @classmethod
  def service_name(cls):
    """
    Particular service implemented by subclasses.
    """
    raise NotImplementedError

  @classmethod
  def get_trigger_for_service(cls, service):
    for subc in cls.__subclasses__():
      if subc.service_name() == service:
        return subc()

    raise InvalidServiceException('Unable to find service: %s' % service)


def raise_unsupported():
  raise io.UnsupportedOperation

def get_trigger_config(trigger):
  try:
    return json.loads(trigger.config)
  except:
    return {}


class GithubBuildTrigger(BuildTrigger):
  """
  BuildTrigger for GitHub that uses the archive API and buildpacks.
  """
  @staticmethod
  def _get_client(auth_token):
    return Github(auth_token,
                  base_url=github_trigger.api_endpoint(),
                  client_id=github_trigger.client_id(),
                  client_secret=github_trigger.client_secret())

  @classmethod
  def service_name(cls):
    return 'github'

  def is_active(self, config):
    return 'hook_id' in config

  def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
    new_build_source = config['build_source']
    gh_client = self._get_client(auth_token)

    # Find the GitHub repository.
    try:
      gh_repo = gh_client.get_repo(new_build_source)
    except UnknownObjectException:
      msg = 'Unable to find GitHub repository for source: %s' % new_build_source
      raise TriggerActivationException(msg)

    # Add a deploy key to the GitHub repository.
    public_key, private_key = generate_ssh_keypair()
    config['credentials'] = [
      {
        'name': 'SSH Public Key',
        'value': public_key,
      },
    ]
    try:
      deploy_key = gh_repo.create_key('%s Builder' % app.config['REGISTRY_TITLE'],
                                      public_key)
      config['deploy_key_id'] = deploy_key.id
    except GithubException:
      msg = 'Unable to add deploy key to repository: %s' % new_build_source
      raise TriggerActivationException(msg)

    # Create a webhook config.
    webhook_config = {
      'url': standard_webhook_url,
      'content_type': 'json',
    }

    # Add the webhook to the GitHub repository.
    try:
      hook = gh_repo.create_hook('web', webhook_config)
      config['hook_id'] = hook.id
      config['master_branch'] = gh_repo.default_branch
    except GithubException:
      msg = 'Unable to create webhook on repository: %s' % new_build_source
      raise TriggerActivationException(msg)

    return config, private_key

  def deactivate(self, auth_token, config):
    gh_client = self._get_client(auth_token)

    # Find the GitHub repository.
    try:
      repo = gh_client.get_repo(config['build_source'])
    except UnknownObjectException:
      msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
      raise TriggerDeactivationException(msg)

    # If the trigger uses a deploy key, remove it.
    try:
      if config['deploy_key_id']:
        deploy_key = repo.get_key(config['deploy_key_id'])
        deploy_key.delete()
    except KeyError:
      # There was no config['deploy_key_id'], thus this is an old trigger without a deploy key.
      pass
    except GithubException:
      msg = 'Unable to remove deploy key: %s' % config['deploy_key_id']
      raise TriggerDeactivationException(msg)

    # Remove the webhook.
    try:
      hook = repo.get_hook(config['hook_id'])
      hook.delete()
    except GithubException:
      msg = 'Unable to remove hook: %s' % config['hook_id']
      raise TriggerDeactivationException(msg)

    config.pop('hook_id', None)

    return config

  def list_build_sources(self, auth_token):
    gh_client = self._get_client(auth_token)
    usr = gh_client.get_user()

    personal = {
      'personal': True,
      'repos': [repo.full_name for repo in usr.get_repos()],
      'info': {
        'name': usr.login,
        'avatar_url': usr.avatar_url,
      }
    }

    repos_by_org = [personal]

    for org in usr.get_orgs():
      repo_list = []
      for repo in org.get_repos(type='member'):
        repo_list.append(repo.full_name)

      repos_by_org.append({
        'personal': False,
        'repos': repo_list,
        'info': {
          'name': org.name or org.login,
          'avatar_url': org.avatar_url
        }
      })

    return repos_by_org

  @staticmethod
  def matches_ref(ref, regex):
    match_string = ref.split('/', 1)[1]
    if not regex:
      return False

    m = regex.match(match_string)
    if not m:
      return False

    return len(m.group(0)) == len(match_string)

  def list_build_subdirs(self, auth_token, config):
    gh_client = self._get_client(auth_token)
    source = config['build_source']

    try:
      repo = gh_client.get_repo(source)

      # Find the first matching branch.
      branches = None
      if 'branchtag_regex' in config:
        try:
          regex = re.compile(config['branchtag_regex'])
          branches = [branch.name for branch in repo.get_branches()
                      if GithubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
        except:
          pass

      branches = branches or [repo.default_branch or 'master']
      default_commit = repo.get_branch(branches[0]).commit
      commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)

      return [os.path.dirname(elem.path) for elem in commit_tree.tree
              if (elem.type == u'blob' and
                  os.path.basename(elem.path) == u'Dockerfile')]
    except GithubException as ge:
      message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
      if message == 'Branch not found':
        raise EmptyRepositoryException()

      raise RepositoryReadException(message)

  def dockerfile_url(self, auth_token, config):
    source = config['build_source']
    subdirectory = config.get('subdir', '')
    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
    gh_client = self._get_client(auth_token)

    try:
      repo = gh_client.get_repo(source)
      master_branch = repo.default_branch or 'master'
      return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
    except GithubException:
      logger.exception('Could not load repository for Dockerfile.')
      return None

  def load_dockerfile_contents(self, auth_token, config):
    gh_client = self._get_client(auth_token)

    source = config['build_source']
    subdirectory = config.get('subdir', '')
    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'

    try:
      repo = gh_client.get_repo(source)
      file_info = repo.get_file_contents(path)
      if file_info is None:
        return None

      content = file_info.content
      if file_info.encoding == 'base64':
        content = base64.b64decode(content)
      return content

    except GithubException as ge:
      message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source)
      raise RepositoryReadException(message)

  @staticmethod
  def _build_commit_info(repo, commit_sha):
    try:
      commit = repo.get_commit(commit_sha)
    except GithubException:
      logger.exception('Could not load data for commit')
      return

    commit_info = {
      'url': commit.html_url,
      'message': commit.commit.message,
      'date': commit.last_modified
    }

    if commit.author:
      commit_info['author'] = {
        'username': commit.author.login,
        'avatar_url': commit.author.avatar_url,
        'url': commit.author.html_url
      }

    if commit.committer:
      commit_info['committer'] = {
        'username': commit.committer.login,
        'avatar_url': commit.committer.avatar_url,
        'url': commit.committer.html_url
      }

    return commit_info

  @staticmethod
  def _prepare_tarball(repo, commit_sha):
    # Prepare the download and upload URLs
    archive_link = repo.get_archive_link('tarball', commit_sha)
    download_archive = client.get(archive_link, stream=True)
    tarball_subdir = ''

    with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
      for chunk in download_archive.iter_content(CHUNK_SIZE):
        tarball.write(chunk)

      # Seek to position 0 to make tarfile happy
      tarball.seek(0)

      # Pull out the name of the subdir that GitHub generated
      with tarfile.open(fileobj=tarball) as archive:
        tarball_subdir = archive.getnames()[0]

      # Seek to position 0 to make tarfile happy.
      tarball.seek(0)

      entries = {
        tarball_subdir + '/.git/HEAD': commit_sha,
        tarball_subdir + '/.git/objects/': None,
        tarball_subdir + '/.git/refs/': None
      }

      appender = TarfileAppender(tarball, entries).get_stream()
      dockerfile_id = user_files.store_file(appender, TARBALL_MIME)

    logger.debug('Successfully prepared job')

    return tarball_subdir, dockerfile_id


  @staticmethod
  def _prepare_build(trigger, config, repo, commit_sha, build_name, ref, git_url):
    repo_subdir = config['subdir']
    joined_subdir = repo_subdir
    dockerfile_id = None

    if trigger.private_key is None:
      # If the trigger isn't using git, prepare the buildpack.
      tarball_subdir, dockerfile_id = GithubBuildTrigger._prepare_tarball(repo, commit_sha)
      logger.debug('Successfully prepared job')

      # Join provided subdir with the tarball subdir.
      joined_subdir = os.path.join(tarball_subdir, repo_subdir)

    logger.debug('Final subdir: %s', joined_subdir)

    # compute the tag(s)
    branch = ref.split('/')[-1]
    tags = {branch}

    if branch == repo.default_branch:
      tags.add('latest')

    logger.debug('Pushing to tags: %s', tags)

    # compute the metadata
    metadata = {
      'commit_sha': commit_sha,
      'ref': ref,
      'default_branch': repo.default_branch,
      'git_url': git_url,
    }

    # add the commit info.
    commit_info = GithubBuildTrigger._build_commit_info(repo, commit_sha)
    if commit_info is not None:
      metadata['commit_info'] = commit_info

    return dockerfile_id, list(tags), build_name, joined_subdir, metadata

  @staticmethod
  def get_display_name(sha):
    return sha[0:7]

  def handle_trigger_request(self, request, trigger):
    payload = request.get_json()
    if not payload or payload.get('head_commit') is None:
      raise SkipRequestException()

    if 'zen' in payload:
      raise ValidationRequestException()

    logger.debug('Payload %s', payload)
    ref = payload['ref']
    commit_sha = payload['head_commit']['id']
    commit_message = payload['head_commit'].get('message', '')
    git_url = payload['repository']['git_url']

    config = get_trigger_config(trigger)
    if 'branchtag_regex' in config:
      try:
        regex = re.compile(config['branchtag_regex'])
      except:
        regex = re.compile('.*')

      if not GithubBuildTrigger.matches_ref(ref, regex):
        raise SkipRequestException()

    if should_skip_commit(commit_message):
      raise SkipRequestException()

    short_sha = GithubBuildTrigger.get_display_name(commit_sha)

    gh_client = self._get_client(trigger.auth_token)

    repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
                                payload['repository']['name'])
    repo = gh_client.get_repo(repo_full_name)

    logger.debug('Github repo: %s', repo)

    return GithubBuildTrigger._prepare_build(trigger, config, repo, commit_sha,
                                             short_sha, ref, git_url)

  def manual_start(self, trigger, run_parameters=None):
    config = get_trigger_config(trigger)
    try:
      source = config['build_source']
      run_parameters = run_parameters or {}

      gh_client = self._get_client(trigger.auth_token)
      repo = gh_client.get_repo(source)
      branch_name = run_parameters.get('branch_name') or repo.default_branch
      branch = repo.get_branch(branch_name)
      branch_sha = branch.commit.sha
      short_sha = GithubBuildTrigger.get_display_name(branch_sha)
      ref = 'refs/heads/%s' % (branch_name)
      git_url = repo.git_url

      return self._prepare_build(trigger, config, repo, branch_sha, short_sha, ref, git_url)
    except GithubException as ghe:
      raise TriggerStartException(ghe.data['message'])


  def list_field_values(self, auth_token, config, field_name):
    if field_name == 'refs':
      branches = self.list_field_values(auth_token, config, 'branch_name')
      tags = self.list_field_values(auth_token, config, 'tag_name')

      return ([{'kind': 'branch', 'name': b} for b in branches] +
              [{'kind': 'tag', 'name': tag} for tag in tags])

    if field_name == 'tag_name':
      gh_client = self._get_client(auth_token)
      source = config['build_source']
      repo = gh_client.get_repo(source)
      return [tag.name for tag in repo.get_tags()]

    if field_name == 'branch_name':
      gh_client = self._get_client(auth_token)
      source = config['build_source']
      repo = gh_client.get_repo(source)
      branches = [branch.name for branch in repo.get_branches()]

      if not repo.default_branch in branches:
        branches.insert(0, repo.default_branch)

      if branches[0] != repo.default_branch:
        branches.remove(repo.default_branch)
        branches.insert(0, repo.default_branch)

      return branches

    return None

class CustomBuildTrigger(BuildTrigger):
  payload_schema = {
    'type': 'object',
    'properties': {
      'commit': {
        'type': 'string',
        'description': 'first 7 characters of the SHA-1 identifier for a git commit',
        'pattern': '^([A-Fa-f0-9]{7})$',
      },
      'ref': {
        'type': 'string',
        'description': 'git reference for a git commit',
        'pattern': '^refs\/(heads|tags|remotes)\/(.+)$',
      },
      'default_branch': {
        'type': 'string',
        'description': 'default branch of the git repository',
      },
      'commit_info': {
        'type': 'object',
        'description': 'metadata about a git commit',
        'properties': {
          'url': {
            'type': 'string',
            'description': 'URL to view a git commit',
          },
          'message': {
            'type': 'string',
            'description': 'git commit message',
          },
          'date': {
            'type': 'string',
            'description': 'timestamp for a git commit'
          },
          'author': {
            'type': 'object',
            'description': 'metadata about the author of a git commit',
            'properties': {
              'username': {
                'type': 'string',
                'description': 'username of the author',
              },
              'url': {
                'type': 'string',
                'description': 'URL to view the profile of the author',
              },
              'avatar_url': {
                'type': 'string',
                'description': 'URL to view the avatar of the author',
              },
            },
            'required': ['username', 'url', 'avatar_url'],
          },
          'committer': {
            'type': 'object',
            'description': 'metadata about the committer of a git commit',
            'properties': {
              'username': {
                'type': 'string',
                'description': 'username of the committer',
              },
              'url': {
                'type': 'string',
                'description': 'URL to view the profile of the committer',
              },
              'avatar_url': {
                'type': 'string',
                'description': 'URL to view the avatar of the committer',
              },
            },
            'required': ['username', 'url', 'avatar_url'],
          },
        },
        'required': ['url', 'message', 'date'],
      },
    },
    'required': ['commits', 'ref', 'default_branch'],
  }

  @classmethod
  def service_name(cls):
    return 'custom-git'

  def is_active(self, config):
    return config.has_key('credentials')

  def _metadata_from_payload(self, payload):
    try:
      metadata = json.loads(payload)
      validate(metadata, self.payload_schema)
    except:
      raise InvalidPayloadException()
    return metadata

  def handle_trigger_request(self, request, trigger):
    payload = request.get_json()
    if not payload:
      raise SkipRequestException()

    logger.debug('Payload %s', payload)
    metadata = self._metadata_from_payload(payload)

    # The build source is the canonical git URL used to clone.
    config = get_trigger_config(trigger)
    metadata['git_url'] = config['build_source']

    branch = metadata['ref'].split('/')[-1]
    tags = {branch}

    build_name = metadata['commit_sha'][:6]
    dockerfile_id = None

    return dockerfile_id, tags, build_name, trigger.config['subdir'], metadata

  def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
    public_key, private_key = generate_ssh_keypair()
    config['credentials'] = [
      {
        'name': 'SSH Public Key',
        'value': public_key,
      },
      {
        'name': 'Webhook Endpoint URL',
        'value': standard_webhook_url,
      },
    ]
    return config, private_key

  def deactivate(self, auth_token, config):
    config.pop('credentials', None)
    return config

  def manual_start(self, trigger, run_parameters=None):
    # commit_sha is the only required parameter
    if 'commit_sha' not in run_parameters:
      raise TriggerStartException('missing required parameter')

    config = get_trigger_config(trigger)
    dockerfile_id = None
    tags = {run_parameters['commit_sha']}
    build_name = run_parameters['commit_sha']
    metadata = {
      'commit_sha': run_parameters['commit_sha'],
      'git_url': config['build_source'],
    }

    return dockerfile_id, list(tags), build_name, config['subdir'], metadata