From 1594b92f71c34e0915aedfb8417e975736ee34f4 Mon Sep 17 00:00:00 2001
From: Jimmy Zelinskie <jimmy.zelinskie@coreos.com>
Date: Wed, 18 Mar 2015 17:33:43 -0400
Subject: [PATCH] endpoints: initial github trigger that uses keys

---
 endpoints/trigger.py | 395 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 384 insertions(+), 11 deletions(-)

diff --git a/endpoints/trigger.py b/endpoints/trigger.py
index bfd9f8ce7..b75cc2952 100644
--- a/endpoints/trigger.py
+++ b/endpoints/trigger.py
@@ -10,6 +10,7 @@ from tempfile import SpooledTemporaryFile
 
 from app import app, userfiles as user_files, github_trigger
 from util.tarfileappender import TarfileAppender
+from util.ssh import generate_ssh_keypair
 
 
 client = app.config['HTTPCLIENT']
@@ -60,7 +61,7 @@ class BuildTrigger(object):
 
   def dockerfile_url(self, auth_token, config):
     """
-    Returns the URL at which the Dockerfile for the trigger can be found or None if none/not applicable.
+    Returns the URL at which the Dockerfile for the trigger is found or None if none/not applicable.
     """
     return None
 
@@ -114,7 +115,7 @@ class BuildTrigger(object):
     """
     raise NotImplementedError
 
-  def manual_start(self, auth_token, config, run_parameters = None):
+  def manual_start(self, auth_token, config, run_parameters=None):
     """
     Manually creates a repository build for this trigger.
     """
@@ -148,6 +149,9 @@ def raise_unsupported():
 
 
 class GithubBuildTrigger(BuildTrigger):
+  """
+  BuildTrigger for GitHub that uses the archive API and buildpacks.
+  """
   @staticmethod
   def _get_client(auth_token):
     return Github(auth_token,
@@ -233,7 +237,8 @@ class GithubBuildTrigger(BuildTrigger):
 
     return repos_by_org
 
-  def matches_ref(self, ref, regex):
+  @staticmethod
+  def matches_ref(ref, regex):
     match_string = ref.split('/', 1)[1]
     if not regex:
       return False
@@ -257,7 +262,7 @@ class GithubBuildTrigger(BuildTrigger):
         try:
           regex = re.compile(config['branchtag_regex'])
           branches = [branch.name for branch in repo.get_branches()
-                                  if self.matches_ref('refs/heads/' + branch.name, regex)]
+                      if GithubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
         except:
           pass
 
@@ -275,7 +280,7 @@ class GithubBuildTrigger(BuildTrigger):
 
       raise RepositoryReadException(message)
 
-  def dockerfile_url(self,  auth_token, config):
+  def dockerfile_url(self, auth_token, config):
     source = config['build_source']
     subdirectory = config.get('subdir', '')
     path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
@@ -285,7 +290,7 @@ class GithubBuildTrigger(BuildTrigger):
       repo = gh_client.get_repo(source)
       master_branch = repo.default_branch or 'master'
       return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
-    except GithubException as ge:
+    except GithubException:
       return None
 
   def load_dockerfile_contents(self, auth_token, config):
@@ -373,12 +378,12 @@ class GithubBuildTrigger(BuildTrigger):
     if branch == repo.default_branch:
       tags.add('latest')
 
-    logger.debug('Pushing to tags: %s' % tags)
+    logger.debug('Pushing to tags: %s', tags)
 
     # compute the subdir
     repo_subdir = config['subdir']
     joined_subdir = os.path.join(tarball_subdir, repo_subdir)
-    logger.debug('Final subdir: %s' % joined_subdir)
+    logger.debug('Final subdir: %s', joined_subdir)
 
     # compute the metadata
     metadata = {
@@ -417,7 +422,7 @@ class GithubBuildTrigger(BuildTrigger):
       except:
         regex = re.compile('.*')
 
-      if not self.matches_ref(ref, regex):
+      if not GithubBuildTrigger.matches_ref(ref, regex):
         raise SkipRequestException()
 
     if should_skip_commit(commit_message):
@@ -436,7 +441,7 @@ class GithubBuildTrigger(BuildTrigger):
     return GithubBuildTrigger._prepare_build(config, repo, commit_sha,
                                              short_sha, ref)
 
-  def manual_start(self, auth_token, config, run_parameters = None):
+  def manual_start(self, auth_token, config, run_parameters=None):
     try:
       source = config['build_source']
       run_parameters = run_parameters or {}
@@ -446,7 +451,6 @@ class GithubBuildTrigger(BuildTrigger):
       branch_name = run_parameters.get('branch_name') or repo.default_branch
       branch = repo.get_branch(branch_name)
       branch_sha = branch.commit.sha
-      commit_info = branch.commit
       short_sha = GithubBuildTrigger.get_display_name(branch_sha)
       ref = 'refs/heads/%s' % (branch_name)
 
@@ -455,6 +459,375 @@ class GithubBuildTrigger(BuildTrigger):
       raise TriggerStartException(ghe.data['message'])
 
 
+  def list_field_values(self, auth_token, config, field_name):
+    if field_name == 'refs':
+      branches = self.list_field_values(auth_token, config, 'branch_name')
+      tags = self.list_field_values(auth_token, config, 'tag_name')
+
+      return ([{'kind': 'branch', 'name': b} for b in branches] +
+              [{'kind': 'tag', 'name': tag} for tag in tags])
+
+    if field_name == 'tag_name':
+      gh_client = self._get_client(auth_token)
+      source = config['build_source']
+      repo = gh_client.get_repo(source)
+      return [tag.name for tag in repo.get_tags()]
+
+    if field_name == 'branch_name':
+      gh_client = self._get_client(auth_token)
+      source = config['build_source']
+      repo = gh_client.get_repo(source)
+      branches = [branch.name for branch in repo.get_branches()]
+
+      if not repo.default_branch in branches:
+        branches.insert(0, repo.default_branch)
+
+      if branches[0] != repo.default_branch:
+        branches.remove(repo.default_branch)
+        branches.insert(0, repo.default_branch)
+
+      return branches
+
+    return None
+
+class GitHubBuildTrigger(BuildTrigger):
+  """
+  BuildTrigger for GitHub that uses deploy keys and git.
+  """
+  @staticmethod
+  def _get_client(auth_token):
+    return Github(auth_token,
+                  base_url=github_trigger.api_endpoint(),
+                  client_id=github_trigger.client_id(),
+                  client_secret=github_trigger.client_secret())
+
+  @classmethod
+  def service_name(cls):
+    return 'github-git'
+
+  def is_active(self, config):
+    return 'hook_id' in config and 'deploy_key_id' in config
+
+  def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
+    new_build_source = config['build_source']
+    gh_client = self._get_client(auth_token)
+
+    # Find the GitHub repository.
+    try:
+      gh_repo = gh_client.get_repo(new_build_source)
+    except UnknownObjectException:
+      msg = 'Unable to find GitHub repository for source: %s' % new_build_source
+      raise TriggerActivationException(msg)
+
+    # Generate an SSH keypair and add the public key to the repository.
+    config['public_key'], config['private_key'] = generate_ssh_keypair()
+    try:
+      deploy_key = gh_repo.create_key('Quay.io Builder', config['public_key'])
+      config['deploy_key_id'] = deploy_key.id
+    except GithubException:
+      msg = 'Unable to add deploy key to repository: %s' % new_build_source
+      raise TriggerActivationException(msg)
+
+    # Create a webhook config.
+    webhook_config = {
+      'url': standard_webhook_url,
+      'content_type': 'json',
+    }
+
+    # Add the webhook to the GitHub repository.
+    try:
+      hook = gh_repo.create_hook('web', webhook_config)
+      config['hook_id'] = hook.id
+      config['master_branch'] = gh_repo.default_branch
+    except GithubException:
+      msg = 'Unable to create webhook on repository: %s' % new_build_source
+      raise TriggerActivationException(msg)
+
+    return config
+
+  def deactivate(self, auth_token, config):
+    gh_client = self._get_client(auth_token)
+
+    # Find the GitHub repository.
+    try:
+      repo = gh_client.get_repo(config['build_source'])
+    except UnknownObjectException:
+      msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
+      raise TriggerDeactivationException(msg)
+
+    # Remove the deploy key.
+    try:
+      deploy_key = repo.get_key(config['deploy_key_id'])
+      deploy_key.delete()
+    except GithubException:
+      msg = 'Unable to remove deploy key to repository: %s' % config['build_source']
+      raise TriggerActivationException(msg)
+    config.pop('deploy_key_id', None)
+
+    # Remove the webhook.
+    try:
+      hook = repo.get_hook(config['hook_id'])
+      hook.delete()
+    except GithubException:
+      msg = 'Unable to remove hook: %s' % config['hook_id']
+      raise TriggerDeactivationException(msg)
+    config.pop('hook_id', None)
+
+    return config
+
+  def list_build_sources(self, auth_token):
+    gh_client = self._get_client(auth_token)
+    usr = gh_client.get_user()
+
+    personal = {
+      'personal': True,
+      'repos': [repo.full_name for repo in usr.get_repos()],
+      'info': {
+        'name': usr.login,
+        'avatar_url': usr.avatar_url,
+      }
+    }
+
+    repos_by_org = [personal]
+
+    for org in usr.get_orgs():
+      repo_list = []
+      for repo in org.get_repos(type='member'):
+        repo_list.append(repo.full_name)
+
+      repos_by_org.append({
+        'personal': False,
+        'repos': repo_list,
+        'info': {
+          'name': org.name or org.login,
+          'avatar_url': org.avatar_url
+        }
+      })
+
+    return repos_by_org
+
+  @staticmethod
+  def matches_ref(ref, regex):
+    match_string = ref.split('/', 1)[1]
+    if not regex:
+      return False
+
+    m = regex.match(match_string)
+    if not m:
+      return False
+
+    return len(m.group(0)) == len(match_string)
+
+  def list_build_subdirs(self, auth_token, config):
+    gh_client = self._get_client(auth_token)
+    source = config['build_source']
+
+    try:
+      repo = gh_client.get_repo(source)
+
+      # Find the first matching branch.
+      branches = None
+      if 'branchtag_regex' in config:
+        try:
+          regex = re.compile(config['branchtag_regex'])
+          branches = [branch.name for branch in repo.get_branches()
+                      if GitHubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
+        except:
+          pass
+
+      branches = branches or [repo.default_branch or 'master']
+      default_commit = repo.get_branch(branches[0]).commit
+      commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
+
+      return [os.path.dirname(elem.path) for elem in commit_tree.tree
+              if (elem.type == u'blob' and
+                  os.path.basename(elem.path) == u'Dockerfile')]
+    except GithubException as ge:
+      message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
+      if message == 'Branch not found':
+        raise EmptyRepositoryException()
+
+      raise RepositoryReadException(message)
+
+  def dockerfile_url(self, auth_token, config):
+    source = config['build_source']
+    subdirectory = config.get('subdir', '')
+    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
+
+    gh_client = self._get_client(auth_token)
+    try:
+      repo = gh_client.get_repo(source)
+      master_branch = repo.default_branch or 'master'
+      return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
+    except GithubException:
+      return None
+
+  def load_dockerfile_contents(self, auth_token, config):
+    gh_client = self._get_client(auth_token)
+
+    source = config['build_source']
+    subdirectory = config.get('subdir', '')
+    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
+
+    try:
+      repo = gh_client.get_repo(source)
+      file_info = repo.get_file_contents(path)
+      if file_info is None:
+        return None
+
+      content = file_info.content
+      if file_info.encoding == 'base64':
+        content = base64.b64decode(content)
+      return content
+
+    except GithubException as ge:
+      message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source)
+      raise RepositoryReadException(message)
+
+  @staticmethod
+  def _build_commit_info(repo, commit_sha):
+    try:
+      commit = repo.get_commit(commit_sha)
+    except GithubException:
+      logger.exception('Could not load data for commit')
+      return
+
+    return {
+      'url': commit.html_url,
+      'message': commit.commit.message,
+      'author': {
+        'username': commit.author.login,
+        'avatar_url': commit.author.avatar_url,
+        'url': commit.author.html_url
+      },
+      'committer': {
+        'username': commit.committer.login,
+        'avatar_url': commit.committer.avatar_url,
+        'url': commit.committer.html_url
+      },
+      'date': commit.last_modified
+    }
+
+  @staticmethod
+  def _prepare_build(config, repo, commit_sha, build_name, ref):
+    # Prepare the download and upload URLs
+    archive_link = repo.get_archive_link('tarball', commit_sha)
+    download_archive = client.get(archive_link, stream=True)
+
+    tarball_subdir = ''
+    with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
+      for chunk in download_archive.iter_content(CHUNK_SIZE):
+        tarball.write(chunk)
+
+      # Seek to position 0 to make tarfile happy
+      tarball.seek(0)
+
+      # Pull out the name of the subdir that GitHub generated
+      with tarfile.open(fileobj=tarball) as archive:
+        tarball_subdir = archive.getnames()[0]
+
+      # Seek to position 0 to make tarfile happy.
+      tarball.seek(0)
+
+      entries = {
+        tarball_subdir + '/.git/HEAD': commit_sha,
+        tarball_subdir + '/.git/objects/': None,
+        tarball_subdir + '/.git/refs/': None
+      }
+
+      appender = TarfileAppender(tarball, entries).get_stream()
+      dockerfile_id = user_files.store_file(appender, TARBALL_MIME)
+
+    logger.debug('Successfully prepared job')
+
+    # compute the tag(s)
+    branch = ref.split('/')[-1]
+    tags = {branch}
+
+    if branch == repo.default_branch:
+      tags.add('latest')
+
+    logger.debug('Pushing to tags: %s', tags)
+
+    # compute the subdir
+    repo_subdir = config['subdir']
+    joined_subdir = os.path.join(tarball_subdir, repo_subdir)
+    logger.debug('Final subdir: %s', joined_subdir)
+
+    # compute the metadata
+    metadata = {
+      'commit_sha': commit_sha,
+      'ref': ref,
+      'default_branch': repo.default_branch,
+    }
+
+    # add the commit info.
+    commit_info = GitHubBuildTrigger._build_commit_info(repo, commit_sha)
+    if commit_info is not None:
+      metadata['commit_info'] = commit_info
+
+    return dockerfile_id, list(tags), build_name, joined_subdir, metadata
+
+  @staticmethod
+  def get_display_name(sha):
+    return sha[0:7]
+
+  def handle_trigger_request(self, request, auth_token, config):
+    payload = request.get_json()
+    if not payload or payload.get('head_commit') is None:
+      raise SkipRequestException()
+
+    if 'zen' in payload:
+      raise ValidationRequestException()
+
+    logger.debug('Payload %s', payload)
+    ref = payload['ref']
+    commit_sha = payload['head_commit']['id']
+    commit_message = payload['head_commit'].get('message', '')
+
+    if 'branchtag_regex' in config:
+      try:
+        regex = re.compile(config['branchtag_regex'])
+      except:
+        regex = re.compile('.*')
+
+      if not GitHubBuildTrigger.matches_ref(ref, regex):
+        raise SkipRequestException()
+
+    if should_skip_commit(commit_message):
+      raise SkipRequestException()
+
+    short_sha = GitHubBuildTrigger.get_display_name(commit_sha)
+
+    gh_client = self._get_client(auth_token)
+
+    repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
+                                payload['repository']['name'])
+    repo = gh_client.get_repo(repo_full_name)
+
+    logger.debug('Github repo: %s', repo)
+
+    return GitHubBuildTrigger._prepare_build(config, repo, commit_sha,
+                                             short_sha, ref)
+
+  def manual_start(self, auth_token, config, run_parameters=None):
+    try:
+      source = config['build_source']
+      run_parameters = run_parameters or {}
+
+      gh_client = self._get_client(auth_token)
+      repo = gh_client.get_repo(source)
+      branch_name = run_parameters.get('branch_name') or repo.default_branch
+      branch = repo.get_branch(branch_name)
+      branch_sha = branch.commit.sha
+      short_sha = GitHubBuildTrigger.get_display_name(branch_sha)
+      ref = 'refs/heads/%s' % (branch_name)
+
+      return self._prepare_build(config, repo, branch_sha, short_sha, ref)
+    except GithubException as ghe:
+      raise TriggerStartException(ghe.data['message'])
+
+
   def list_field_values(self, auth_token, config, field_name):
     if field_name == 'refs':
       branches = self.list_field_values(auth_token, config, 'branch_name')