diff --git a/data/database.py b/data/database.py index 4af55e044..a7755c9bf 100644 --- a/data/database.py +++ b/data/database.py @@ -471,6 +471,9 @@ class RepositoryBuildTrigger(BaseModel): pull_robot = QuayUserField(allows_robots=True, null=True, related_name='triggerpullrobot', robot_null_delete=True) + # TODO(jschorr): Remove this column once we verify the backfill has succeeded. + used_legacy_github = BooleanField(null=True, default=False) + class EmailConfirmation(BaseModel): code = CharField(default=random_string_generator(), unique=True, index=True) diff --git a/data/migrations/versions/3ff4fbc94644_migrate_github_triggers_to_use_deploy_.py b/data/migrations/versions/3ff4fbc94644_migrate_github_triggers_to_use_deploy_.py new file mode 100644 index 000000000..820b21548 --- /dev/null +++ b/data/migrations/versions/3ff4fbc94644_migrate_github_triggers_to_use_deploy_.py @@ -0,0 +1,28 @@ +"""Migrate GitHub triggers to use deploy keys + +Revision ID: 3ff4fbc94644 +Revises: 4d5f6716df0 +Create Date: 2015-09-16 17:50:22.034146 + +""" + +# revision identifiers, used by Alembic. +revision = '3ff4fbc94644' +down_revision = '4d5f6716df0' + +from alembic import op +import sqlalchemy as sa + +from util.migrate.migrategithubdeploykeys import backfill_github_deploykeys + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + backfill_github_deploykeys() + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + pass + ### end Alembic commands ### diff --git a/data/migrations/versions/4d5f6716df0_add_legacy_column_for_github_backfill_.py b/data/migrations/versions/4d5f6716df0_add_legacy_column_for_github_backfill_.py new file mode 100644 index 000000000..94a036548 --- /dev/null +++ b/data/migrations/versions/4d5f6716df0_add_legacy_column_for_github_backfill_.py @@ -0,0 +1,26 @@ +"""Add legacy column for GitHub backfill tracking + +Revision ID: 4d5f6716df0 +Revises: 545794454f49 +Create Date: 2015-09-16 17:49:40.334540 + +""" + +# revision identifiers, used by Alembic. +revision = '4d5f6716df0' +down_revision = '545794454f49' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('repositorybuildtrigger', sa.Column('used_legacy_github', sa.Boolean(), nullable=True)) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('repositorybuildtrigger', 'used_legacy_github') + ### end Alembic commands ### diff --git a/util/migrate/migratebitbucketservices.py b/util/migrate/migratebitbucketservices.py index 880f5308e..c342654e7 100644 --- a/util/migrate/migratebitbucketservices.py +++ b/util/migrate/migratebitbucketservices.py @@ -2,7 +2,8 @@ import logging import json from app import app -from data.database import configure, RepositoryBuildTrigger, BuildTriggerService +from data.database import configure, BaseModel, uuid_generator +from peewee import * from bitbucket import BitBucket from endpoints.trigger import BitbucketBuildTrigger @@ -10,6 +11,31 @@ configure(app.config) logger = logging.getLogger(__name__) +# Note: We vendor the RepositoryBuildTrigger and its dependencies here +class Repository(BaseModel): + pass + +class BuildTriggerService(BaseModel): + name = CharField(index=True, unique=True) + +class AccessToken(BaseModel): + pass + +class User(BaseModel): + pass + +class RepositoryBuildTrigger(BaseModel): + uuid = CharField(default=uuid_generator) + service = ForeignKeyField(BuildTriggerService, index=True) + repository = ForeignKeyField(Repository, index=True) + connected_user = ForeignKeyField(User) + auth_token = CharField(null=True) + private_key = TextField(null=True) + config = TextField(default='{}') + write_token = ForeignKeyField(AccessToken, null=True) + pull_robot = ForeignKeyField(User, related_name='triggerpullrobot') + + def run_bitbucket_migration(): bitbucket_trigger = BuildTriggerService.get(BuildTriggerService.name == "bitbucket") diff --git a/util/migrate/migrategithubdeploykeys.py b/util/migrate/migrategithubdeploykeys.py new file mode 100644 index 000000000..2d717f74e --- /dev/null +++ b/util/migrate/migrategithubdeploykeys.py @@ -0,0 +1,87 @@ +import logging +import logging.config +import json + +from data.database import RepositoryBuildTrigger, BuildTriggerService, db, db_for_update +from app import app +from endpoints.trigger import BuildTriggerHandler +from util.security.ssh import generate_ssh_keypair +from github import GithubException + +logger = logging.getLogger(__name__) + +def backfill_github_deploykeys(): + """ Generates and saves private deploy keys for any GitHub build triggers still relying on + the old buildpack behavior. """ + logger.setLevel(logging.DEBUG) + logger.debug('GitHub deploy key backfill: Began execution') + + encountered = set() + github_service = BuildTriggerService.get(name='github') + + while True: + build_trigger_ids = list(RepositoryBuildTrigger + .select(RepositoryBuildTrigger.id) + .where(RepositoryBuildTrigger.private_key >> None) + .where(RepositoryBuildTrigger.service == github_service) + .limit(10)) + + filtered_ids = [trigger.id for trigger in build_trigger_ids if trigger.id not in encountered] + if len(filtered_ids) == 0: + # We're done! + logger.debug('GitHub deploy key backfill: Backfill completed') + return + + logger.debug('GitHub deploy key backfill: Found %s records to update', len(filtered_ids)) + for trigger_id in filtered_ids: + encountered.add(trigger_id) + logger.debug('Updating build trigger: %s', trigger_id) + + with app.config['DB_TRANSACTION_FACTORY'](db): + try: + query = RepositoryBuildTrigger.select(RepositoryBuildTrigger.id == trigger_id) + trigger = db_for_update(query).get() + except RepositoryBuildTrigger.DoesNotExist: + logger.debug('Could not find build trigger %s', trigger_id) + continue + + handler = BuildTriggerHandler.get_handler(trigger) + + config = handler.config + build_source = config['build_source'] + gh_client = handler._get_client() + + # Find the GitHub repository. + try: + gh_repo = gh_client.get_repo(build_source) + except GithubException: + logger.exception('Cannot find repository %s for trigger %s', build_source, trigger.id) + continue + + # Add a deploy key to the GitHub repository. + public_key, private_key = generate_ssh_keypair() + config['credentials'] = [ + { + 'name': 'SSH Public Key', + 'value': public_key, + }, + ] + + logger.debug('Adding deploy key to build trigger %s', trigger.id) + try: + deploy_key = gh_repo.create_key('%s Builder' % app.config['REGISTRY_TITLE'], public_key) + config['deploy_key_id'] = deploy_key.id + except GithubException: + logger.exception('Cannot add deploy key to repository %s for trigger %s', build_source, trigger.id) + continue + + logger.debug('Saving deploy key for trigger %s', trigger.id) + trigger.used_legacy_github = True + trigger.private_key = private_key + trigger.config = json.dumps(config) + trigger.save() + + +if __name__ == "__main__": + logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False) + backfill_github_deploykeys()