Migrate all GitHub build triggers to use deploy keys

data/database.py

@@ -471,6 +471,9 @@ class RepositoryBuildTrigger(BaseModel):
   pull_robot = QuayUserField(allows_robots=True, null=True, related_name='triggerpullrobot',
                              robot_null_delete=True)
 
+  # TODO(jschorr): Remove this column once we verify the backfill has succeeded.
+  used_legacy_github = BooleanField(null=True, default=False)
+
 
 class EmailConfirmation(BaseModel):
   code = CharField(default=random_string_generator(), unique=True, index=True)

data/migrations/versions/3ff4fbc94644_migrate_github_triggers_to_use_deploy_.py

@@ -0,0 +1,28 @@
+"""Migrate GitHub triggers to use deploy keys
+
+Revision ID: 3ff4fbc94644
+Revises: 4d5f6716df0
+Create Date: 2015-09-16 17:50:22.034146
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3ff4fbc94644'
+down_revision = '4d5f6716df0'
+
+from alembic import op
+import sqlalchemy as sa
+
+from util.migrate.migrategithubdeploykeys import backfill_github_deploykeys
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    backfill_github_deploykeys()
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    pass
+    ### end Alembic commands ###

data/migrations/versions/4d5f6716df0_add_legacy_column_for_github_backfill_.py

@@ -0,0 +1,26 @@
+"""Add legacy column for GitHub backfill tracking
+
+Revision ID: 4d5f6716df0
+Revises: 545794454f49
+Create Date: 2015-09-16 17:49:40.334540
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4d5f6716df0'
+down_revision = '545794454f49'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('repositorybuildtrigger', sa.Column('used_legacy_github', sa.Boolean(), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('repositorybuildtrigger', 'used_legacy_github')
+    ### end Alembic commands ###

util/migrate/migratebitbucketservices.py

@@ -2,7 +2,8 @@ import logging
 import json
 
 from app import app
-from data.database import configure, RepositoryBuildTrigger, BuildTriggerService
+from data.database import configure, BaseModel, uuid_generator
+from peewee import *
 from bitbucket import BitBucket
 from endpoints.trigger import BitbucketBuildTrigger
 
@@ -10,6 +11,31 @@ configure(app.config)
 
 logger = logging.getLogger(__name__)
 
+# Note: We vendor the RepositoryBuildTrigger and its dependencies here
+class Repository(BaseModel):
+  pass
+
+class BuildTriggerService(BaseModel):
+  name = CharField(index=True, unique=True)
+
+class AccessToken(BaseModel):
+  pass
+
+class User(BaseModel):
+  pass
+
+class RepositoryBuildTrigger(BaseModel):
+  uuid = CharField(default=uuid_generator)
+  service = ForeignKeyField(BuildTriggerService, index=True)
+  repository = ForeignKeyField(Repository, index=True)
+  connected_user = ForeignKeyField(User)
+  auth_token = CharField(null=True)
+  private_key = TextField(null=True)
+  config = TextField(default='{}')
+  write_token = ForeignKeyField(AccessToken, null=True)
+  pull_robot = ForeignKeyField(User, related_name='triggerpullrobot')
+
+
 def run_bitbucket_migration():
   bitbucket_trigger = BuildTriggerService.get(BuildTriggerService.name == "bitbucket")
 

util/migrate/migrategithubdeploykeys.py

@@ -0,0 +1,87 @@
+import logging
+import logging.config
+import json
+
+from data.database import RepositoryBuildTrigger, BuildTriggerService, db, db_for_update
+from app import app
+from endpoints.trigger import BuildTriggerHandler
+from util.security.ssh import generate_ssh_keypair
+from github import GithubException
+
+logger = logging.getLogger(__name__)
+
+def backfill_github_deploykeys():
+  """ Generates and saves private deploy keys for any GitHub build triggers still relying on
+      the old buildpack behavior. """
+  logger.setLevel(logging.DEBUG)
+  logger.debug('GitHub deploy key backfill: Began execution')
+
+  encountered = set()
+  github_service = BuildTriggerService.get(name='github')
+
+  while True:
+    build_trigger_ids = list(RepositoryBuildTrigger
+                              .select(RepositoryBuildTrigger.id)
+                              .where(RepositoryBuildTrigger.private_key >> None)
+                              .where(RepositoryBuildTrigger.service == github_service)
+                              .limit(10))
+
+    filtered_ids = [trigger.id for trigger in build_trigger_ids if trigger.id not in encountered]
+    if len(filtered_ids) == 0:
+      # We're done!
+      logger.debug('GitHub deploy key backfill: Backfill completed')
+      return
+
+    logger.debug('GitHub deploy key backfill: Found %s records to update', len(filtered_ids))
+    for trigger_id in filtered_ids:
+      encountered.add(trigger_id)
+      logger.debug('Updating build trigger: %s', trigger_id)
+
+      with app.config['DB_TRANSACTION_FACTORY'](db):
+        try:
+          query = RepositoryBuildTrigger.select(RepositoryBuildTrigger.id == trigger_id)
+          trigger = db_for_update(query).get()
+        except RepositoryBuildTrigger.DoesNotExist:
+          logger.debug('Could not find build trigger %s', trigger_id)
+          continue
+
+        handler = BuildTriggerHandler.get_handler(trigger)
+
+        config = handler.config
+        build_source = config['build_source']
+        gh_client = handler._get_client()
+
+        # Find the GitHub repository.
+        try:
+          gh_repo = gh_client.get_repo(build_source)
+        except GithubException:
+          logger.exception('Cannot find repository %s for trigger %s', build_source, trigger.id)
+          continue
+
+        # Add a deploy key to the GitHub repository.
+        public_key, private_key = generate_ssh_keypair()
+        config['credentials'] = [
+          {
+            'name': 'SSH Public Key',
+            'value': public_key,
+          },
+        ]
+
+        logger.debug('Adding deploy key to build trigger %s', trigger.id)
+        try:
+          deploy_key = gh_repo.create_key('%s Builder' % app.config['REGISTRY_TITLE'], public_key)
+          config['deploy_key_id'] = deploy_key.id
+        except GithubException:
+          logger.exception('Cannot add deploy key to repository %s for trigger %s', build_source, trigger.id)
+          continue
+
+        logger.debug('Saving deploy key for trigger %s', trigger.id)
+        trigger.used_legacy_github = True
+        trigger.private_key = private_key
+        trigger.config = json.dumps(config)
+        trigger.save()
+
+
+if __name__ == "__main__":
+  logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
+  backfill_github_deploykeys()