consolidate everything into one GitHub trigger

endpoints/callbacks.py

@@ -248,8 +248,6 @@ def github_oauth_attach():
 
 @callback.route('/github/callback/trigger/<path:repository>', methods=['GET'])
 @callback.route('/github/callback/trigger/<path:repository>/__new', methods=['GET'])
-@callback.route('/github/callback/trigger/<path:repostiory>/__git', methods=['GET'])
-@callback.route('/github/callback/trigger/<path:repostiory>/__git/__new', methods=['GET'])
 @route_show_if(features.GITHUB_BUILD)
 @require_session_login
 @parse_repository_name
@@ -262,8 +260,7 @@ def attach_github_build_trigger(namespace, repository):
       msg = 'Invalid repository: %s/%s' % (namespace, repository)
       abort(404, message=msg)
 
-    service_name = 'github-git' if '__git' in request.url else 'github'
+    trigger = model.create_build_trigger(repo, 'github', token, current_user.db_user())
-    trigger = model.create_build_trigger(repo, service_name, token, current_user.db_user())
 
     # TODO(jschorr): Remove once the new layout is in place.
     admin_path = '%s/%s/%s' % (namespace, repository, 'admin')

endpoints/trigger.py

@@ -169,34 +169,61 @@ class GithubBuildTrigger(BuildTrigger):
     new_build_source = config['build_source']
     gh_client = self._get_client(auth_token)
 
+    # Find the GitHub repository.
     try:
-      to_add_webhook = gh_client.get_repo(new_build_source)
+      gh_repo = gh_client.get_repo(new_build_source)
     except UnknownObjectException:
-      msg = 'Unable to find GitHub repository for source: %s'
+      msg = 'Unable to find GitHub repository for source: %s' % new_build_source
-      raise TriggerActivationException(msg % new_build_source)
+      raise TriggerActivationException(msg)
 
+    # Add a deploy key to the GitHub repository.
+    try:
+      deploy_key = gh_repo.create_key('Quay.io Builder', config['public_key'])
+      config['deploy_key_id'] = deploy_key.id
+    except GithubException:
+      msg = 'Unable to add deploy key to repository: %s' % new_build_source
+      raise TriggerActivationException(msg)
+
+    # Create a webhook config.
     webhook_config = {
       'url': standard_webhook_url,
       'content_type': 'json',
     }
 
+    # Add the webhook to the GitHub repository.
     try:
-      hook = to_add_webhook.create_hook('web', webhook_config)
+      hook = gh_repo.create_hook('web', webhook_config)
       config['hook_id'] = hook.id
-      config['master_branch'] = to_add_webhook.default_branch
+      config['master_branch'] = gh_repo.default_branch
     except GithubException:
-      msg = 'Unable to create webhook on repository: %s'
+      msg = 'Unable to create webhook on repository: %s' % new_build_source
-      raise TriggerActivationException(msg % new_build_source)
+      raise TriggerActivationException(msg)
 
     return config
 
   def deactivate(self, auth_token, config):
     gh_client = self._get_client(auth_token)
 
+    # Find the GitHub repository.
     try:
       repo = gh_client.get_repo(config['build_source'])
-      to_delete = repo.get_hook(config['hook_id'])
+    except UnknownObjectException:
-      to_delete.delete()
+      msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
+      raise TriggerDeactivationException(msg)
+
+    # If the trigger uses a deploy key, remove it.
+    if config['deploy_key_id']:
+      try:
+        deploy_key = repo.get_key(config['deploy_key_id'])
+        deploy_key.delete()
+      except GithubException:
+        msg = 'Unable to remove deploy key: %s' % config['deploy_key_id']
+        raise TriggerDeactivationException(msg)
+
+    # Remove the webhook.
+    try:
+      hook = repo.get_hook(config['hook_id'])
+      hook.delete()
     except GithubException:
       msg = 'Unable to remove hook: %s' % config['hook_id']
       raise TriggerDeactivationException(msg)
@@ -283,8 +310,8 @@ class GithubBuildTrigger(BuildTrigger):
     source = config['build_source']
     subdirectory = config.get('subdir', '')
     path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
-
     gh_client = self._get_client(auth_token)
+
     try:
       repo = gh_client.get_repo(source)
       master_branch = repo.default_branch or 'master'
@@ -464,374 +491,6 @@ class GithubBuildTrigger(BuildTrigger):
       raise TriggerStartException(ghe.data['message'])
 
 
-  def list_field_values(self, auth_token, config, field_name):
-    if field_name == 'refs':
-      branches = self.list_field_values(auth_token, config, 'branch_name')
-      tags = self.list_field_values(auth_token, config, 'tag_name')
-
-      return ([{'kind': 'branch', 'name': b} for b in branches] +
-              [{'kind': 'tag', 'name': tag} for tag in tags])
-
-    if field_name == 'tag_name':
-      gh_client = self._get_client(auth_token)
-      source = config['build_source']
-      repo = gh_client.get_repo(source)
-      return [tag.name for tag in repo.get_tags()]
-
-    if field_name == 'branch_name':
-      gh_client = self._get_client(auth_token)
-      source = config['build_source']
-      repo = gh_client.get_repo(source)
-      branches = [branch.name for branch in repo.get_branches()]
-
-      if not repo.default_branch in branches:
-        branches.insert(0, repo.default_branch)
-
-      if branches[0] != repo.default_branch:
-        branches.remove(repo.default_branch)
-        branches.insert(0, repo.default_branch)
-
-      return branches
-
-    return None
-
-class GitHubBuildTrigger(BuildTrigger):
-  """
-  BuildTrigger for GitHub that uses deploy keys and git.
-  """
-  @staticmethod
-  def _get_client(auth_token):
-    return Github(auth_token,
-                  base_url=github_trigger.api_endpoint(),
-                  client_id=github_trigger.client_id(),
-                  client_secret=github_trigger.client_secret())
-
-  @classmethod
-  def service_name(cls):
-    return 'github-git'
-
-  def is_active(self, config):
-    return 'hook_id' in config and 'deploy_key_id' in config
-
-  def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
-    new_build_source = config['build_source']
-    gh_client = self._get_client(auth_token)
-
-    # Find the GitHub repository.
-    try:
-      gh_repo = gh_client.get_repo(new_build_source)
-    except UnknownObjectException:
-      msg = 'Unable to find GitHub repository for source: %s' % new_build_source
-      raise TriggerActivationException(msg)
-
-    # Add a deploy key to the GitHub repository.
-    try:
-      deploy_key = gh_repo.create_key('Quay.io Builder', config['public_key'])
-      config['deploy_key_id'] = deploy_key.id
-    except GithubException:
-      msg = 'Unable to add deploy key to repository: %s' % new_build_source
-      raise TriggerActivationException(msg)
-
-    # Create a webhook config.
-    webhook_config = {
-      'url': standard_webhook_url,
-      'content_type': 'json',
-    }
-
-    # Add the webhook to the GitHub repository.
-    try:
-      hook = gh_repo.create_hook('web', webhook_config)
-      config['hook_id'] = hook.id
-      config['master_branch'] = gh_repo.default_branch
-    except GithubException:
-      msg = 'Unable to create webhook on repository: %s' % new_build_source
-      raise TriggerActivationException(msg)
-
-    return config
-
-  def deactivate(self, auth_token, config):
-    gh_client = self._get_client(auth_token)
-
-    # Find the GitHub repository.
-    try:
-      repo = gh_client.get_repo(config['build_source'])
-    except UnknownObjectException:
-      msg = 'Unable to find GitHub repository for source: %s' % config['build_source']
-      raise TriggerDeactivationException(msg)
-
-    # Remove the deploy key.
-    try:
-      deploy_key = repo.get_key(config['deploy_key_id'])
-      deploy_key.delete()
-    except GithubException:
-      msg = 'Unable to remove deploy key to repository: %s' % config['build_source']
-      raise TriggerActivationException(msg)
-    config.pop('deploy_key_id', None)
-
-    # Remove the webhook.
-    try:
-      hook = repo.get_hook(config['hook_id'])
-      hook.delete()
-    except GithubException:
-      msg = 'Unable to remove hook: %s' % config['hook_id']
-      raise TriggerDeactivationException(msg)
-    config.pop('hook_id', None)
-
-    return config
-
-  def list_build_sources(self, auth_token):
-    gh_client = self._get_client(auth_token)
-    usr = gh_client.get_user()
-
-    personal = {
-      'personal': True,
-      'repos': [repo.full_name for repo in usr.get_repos()],
-      'info': {
-        'name': usr.login,
-        'avatar_url': usr.avatar_url,
-      }
-    }
-
-    repos_by_org = [personal]
-
-    for org in usr.get_orgs():
-      repo_list = []
-      for repo in org.get_repos(type='member'):
-        repo_list.append(repo.full_name)
-
-      repos_by_org.append({
-        'personal': False,
-        'repos': repo_list,
-        'info': {
-          'name': org.name or org.login,
-          'avatar_url': org.avatar_url
-        }
-      })
-
-    return repos_by_org
-
-  @staticmethod
-  def matches_ref(ref, regex):
-    match_string = ref.split('/', 1)[1]
-    if not regex:
-      return False
-
-    m = regex.match(match_string)
-    if not m:
-      return False
-
-    return len(m.group(0)) == len(match_string)
-
-  def list_build_subdirs(self, auth_token, config):
-    gh_client = self._get_client(auth_token)
-    source = config['build_source']
-
-    try:
-      repo = gh_client.get_repo(source)
-
-      # Find the first matching branch.
-      branches = None
-      if 'branchtag_regex' in config:
-        try:
-          regex = re.compile(config['branchtag_regex'])
-          branches = [branch.name for branch in repo.get_branches()
-                      if GitHubBuildTrigger.matches_ref('refs/heads/' + branch.name, regex)]
-        except:
-          pass
-
-      branches = branches or [repo.default_branch or 'master']
-      default_commit = repo.get_branch(branches[0]).commit
-      commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
-
-      return [os.path.dirname(elem.path) for elem in commit_tree.tree
-              if (elem.type == u'blob' and
-                  os.path.basename(elem.path) == u'Dockerfile')]
-    except GithubException as ge:
-      message = ge.data.get('message', 'Unable to list contents of repository: %s' % source)
-      if message == 'Branch not found':
-        raise EmptyRepositoryException()
-
-      raise RepositoryReadException(message)
-
-  def dockerfile_url(self, auth_token, config):
-    source = config['build_source']
-    subdirectory = config.get('subdir', '')
-    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
-
-    gh_client = self._get_client(auth_token)
-    try:
-      repo = gh_client.get_repo(source)
-      master_branch = repo.default_branch or 'master'
-      return 'https://github.com/%s/blob/%s/%s' % (source, master_branch, path)
-    except GithubException:
-      return None
-
-  def load_dockerfile_contents(self, auth_token, config):
-    gh_client = self._get_client(auth_token)
-
-    source = config['build_source']
-    subdirectory = config.get('subdir', '')
-    path = subdirectory + '/Dockerfile' if subdirectory else 'Dockerfile'
-
-    try:
-      repo = gh_client.get_repo(source)
-      file_info = repo.get_file_contents(path)
-      if file_info is None:
-        return None
-
-      content = file_info.content
-      if file_info.encoding == 'base64':
-        content = base64.b64decode(content)
-      return content
-
-    except GithubException as ge:
-      message = ge.data.get('message', 'Unable to read Dockerfile: %s' % source)
-      raise RepositoryReadException(message)
-
-  @staticmethod
-  def _build_commit_info(repo, commit_sha):
-    try:
-      commit = repo.get_commit(commit_sha)
-    except GithubException:
-      logger.exception('Could not load data for commit')
-      return
-
-    return {
-      'url': commit.html_url,
-      'message': commit.commit.message,
-      'author': {
-        'username': commit.author.login,
-        'avatar_url': commit.author.avatar_url,
-        'url': commit.author.html_url
-      },
-      'committer': {
-        'username': commit.committer.login,
-        'avatar_url': commit.committer.avatar_url,
-        'url': commit.committer.html_url
-      },
-      'date': commit.last_modified
-    }
-
-  @staticmethod
-  def _prepare_build(config, repo, commit_sha, build_name, ref):
-    # Prepare the download and upload URLs
-    archive_link = repo.get_archive_link('tarball', commit_sha)
-    download_archive = client.get(archive_link, stream=True)
-
-    tarball_subdir = ''
-    with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
-      for chunk in download_archive.iter_content(CHUNK_SIZE):
-        tarball.write(chunk)
-
-      # Seek to position 0 to make tarfile happy
-      tarball.seek(0)
-
-      # Pull out the name of the subdir that GitHub generated
-      with tarfile.open(fileobj=tarball) as archive:
-        tarball_subdir = archive.getnames()[0]
-
-      # Seek to position 0 to make tarfile happy.
-      tarball.seek(0)
-
-      entries = {
-        tarball_subdir + '/.git/HEAD': commit_sha,
-        tarball_subdir + '/.git/objects/': None,
-        tarball_subdir + '/.git/refs/': None
-      }
-
-      appender = TarfileAppender(tarball, entries).get_stream()
-      dockerfile_id = user_files.store_file(appender, TARBALL_MIME)
-
-    logger.debug('Successfully prepared job')
-
-    # compute the tag(s)
-    branch = ref.split('/')[-1]
-    tags = {branch}
-
-    if branch == repo.default_branch:
-      tags.add('latest')
-
-    logger.debug('Pushing to tags: %s', tags)
-
-    # compute the subdir
-    repo_subdir = config['subdir']
-    joined_subdir = os.path.join(tarball_subdir, repo_subdir)
-    logger.debug('Final subdir: %s', joined_subdir)
-
-    # compute the metadata
-    metadata = {
-      'commit_sha': commit_sha,
-      'ref': ref,
-      'default_branch': repo.default_branch,
-    }
-
-    # add the commit info.
-    commit_info = GitHubBuildTrigger._build_commit_info(repo, commit_sha)
-    if commit_info is not None:
-      metadata['commit_info'] = commit_info
-
-    return dockerfile_id, list(tags), build_name, joined_subdir, metadata
-
-  @staticmethod
-  def get_display_name(sha):
-    return sha[0:7]
-
-  def handle_trigger_request(self, request, auth_token, config):
-    payload = request.get_json()
-    if not payload or payload.get('head_commit') is None:
-      raise SkipRequestException()
-
-    if 'zen' in payload:
-      raise ValidationRequestException()
-
-    logger.debug('Payload %s', payload)
-    ref = payload['ref']
-    commit_sha = payload['head_commit']['id']
-    commit_message = payload['head_commit'].get('message', '')
-
-    if 'branchtag_regex' in config:
-      try:
-        regex = re.compile(config['branchtag_regex'])
-      except:
-        regex = re.compile('.*')
-
-      if not GitHubBuildTrigger.matches_ref(ref, regex):
-        raise SkipRequestException()
-
-    if should_skip_commit(commit_message):
-      raise SkipRequestException()
-
-    short_sha = GitHubBuildTrigger.get_display_name(commit_sha)
-
-    gh_client = self._get_client(auth_token)
-
-    repo_full_name = '%s/%s' % (payload['repository']['owner']['name'],
-                                payload['repository']['name'])
-    repo = gh_client.get_repo(repo_full_name)
-
-    logger.debug('Github repo: %s', repo)
-
-    return GitHubBuildTrigger._prepare_build(config, repo, commit_sha,
-                                             short_sha, ref)
-
-  def manual_start(self, auth_token, config, run_parameters=None):
-    try:
-      source = config['build_source']
-      run_parameters = run_parameters or {}
-
-      gh_client = self._get_client(auth_token)
-      repo = gh_client.get_repo(source)
-      branch_name = run_parameters.get('branch_name') or repo.default_branch
-      branch = repo.get_branch(branch_name)
-      branch_sha = branch.commit.sha
-      short_sha = GitHubBuildTrigger.get_display_name(branch_sha)
-      ref = 'refs/heads/%s' % (branch_name)
-
-      return self._prepare_build(config, repo, branch_sha, short_sha, ref)
-    except GithubException as ghe:
-      raise TriggerStartException(ghe.data['message'])
-
-
   def list_field_values(self, auth_token, config, field_name):
     if field_name == 'refs':
       branches = self.list_field_values(auth_token, config, 'branch_name')

initdb.py

@@ -195,7 +195,6 @@ def initialize_database():
   LoginService.create(name='ldap')
 
   BuildTriggerService.create(name='github')
-  BuildTriggerService.create(name='github-git')
 
   AccessTokenKind.create(name='build-worker')
   AccessTokenKind.create(name='pushpull-token')

static/js/services/trigger-service.js

@@ -8,45 +8,6 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
 
   var triggerTypes = {
     'github': {
-      'description': function(config) {
-        var source = UtilService.textToSafeHtml(config['build_source']);
-        var desc = '<i class="fa fa-github fa-lg" style="margin-left: 2px; margin-right: 2px"></i> Push to Github Repository (Deprecated) ';
-        desc += '<a href="https://github.com/' + source  + '" target="_blank">' + source + '</a>';
-        desc += '<br>Dockerfile folder: //' + UtilService.textToSafeHtml(config['subdir']);
-        return desc;
-      },
-      'run_parameters': [
-        {
-          'title': 'Branch',
-          'type': 'option',
-          'name': 'branch_name'
-        }
-      ],
-      'get_redirect_url': function(namespace, repository) {
-        var redirect_uri = KeyService['githubRedirectUri'] + '/trigger/' +
-          namespace + '/' + repository;
-
-        var authorize_url = KeyService['githubTriggerAuthorizeUrl'];
-        var client_id = KeyService['githubTriggerClientId'];
-
-        return authorize_url + 'client_id=' + client_id +
-          '&scope=repo,user:email&redirect_uri=' + redirect_uri;
-      },
-      'is_enabled': function() {
-        return Features.GITHUB_BUILD;
-      },
-      'icon': 'fa-github',
-      'title': function() {
-        var isEnterprise = KeyService.isEnterprise('github-trigger');
-        if (isEnterprise) {
-          return 'GitHub Enterprise Repository Push (API)';
-        }
-
-        return 'GitHub Repository Push (API)';
-      }
-    },
-
-    'github-git': {
       'description': function(config) {
         var source = UtilService.textToSafeHtml(config['build_source']);
         var desc = '<i class="fa fa-github fa-lg" style="margin-left: 2px; margin-right: 2px"></i> Push to Github Repository ';
@@ -63,7 +24,7 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
       ],
       'get_redirect_url': function(namespace, repository) {
         var redirect_uri = KeyService['githubRedirectUri'] + '/trigger/' +
-          namespace + '/' + repository + '/__git';
+          namespace + '/' + repository;
 
         // TODO(jschorr): Remove once the new layout is in place.
         if (CookieService.get('quay.exp-new-layout') == 'true') {
@@ -83,10 +44,10 @@ angular.module('quay').factory('TriggerService', ['UtilService', '$sanitize', 'K
       'title': function() {
         var isEnterprise = KeyService.isEnterprise('github-trigger');
         if (isEnterprise) {
-          return 'GitHub Enterprise Repository Push (Deploy Key)';
+          return 'GitHub Enterprise Repository Push';
         }
 
-        return 'GitHub Repository Push (Deploy Key)';
+        return 'GitHub Repository Push';
       }
     }
   }

util/ssh.py

@@ -5,6 +5,6 @@ def generate_ssh_keypair():
   Generates a new 2048 bit RSA public key in OpenSSH format and private key in PEM format.
   """
   key = RSA.generate(2048)
-  public_key = key.publicKey().exportKey('OpenSSH')
+  public_key = key.publickey().exportKey('OpenSSH')
   private_key = key.exportKey('PEM')
   return (public_key, private_key)