Merge pull request #256 from coreos-inc/betterbbmigrate

Bitbucket migration

data/migrations/versions/437ee6269a9d_migrate_bitbucket_services_to_webhooks.py

@@ -0,0 +1,24 @@
+"""Migrate BitBucket services to webhooks
+
+Revision ID: 437ee6269a9d
+Revises: 154f2befdfbe
+Create Date: 2015-07-21 14:03:44.964200
+
+"""
+
+from util.migratebitbucketservices import run_bitbucket_migration
+
+# revision identifiers, used by Alembic.
+revision = '437ee6269a9d'
+down_revision = '2e09ad97b06c'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    run_bitbucket_migration()
+
+
+def downgrade(tables):
+    pass

endpoints/trigger.py

@@ -359,19 +359,10 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
   def deactivate(self):
     config = self.config
 
-    # TODO(jschorr): Remove the old hook removal code once everyone is migrated.
-    old_hook_id = config.pop('hook_id', None)
     webhook_id = config.pop('webhook_id', None)
     deploy_key_id = config.pop('deploy_key_id', None)
     repository = self._get_repository_client()
 
-    # Remove the old service hook.
-    if old_hook_id is not None:
-      (result, _, err_msg) = repository.services().delete(old_hook_id)
-      if not result:
-        msg = 'Unable to remove service hook from repository: %s' % err_msg
-        raise TriggerDeactivationException(msg)
-
     # Remove the webhook.
     if webhook_id is not None:
       (result, _, err_msg) = repository.webhooks().delete(webhook_id)
@@ -556,15 +547,6 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
     return prepared
 
   def handle_trigger_request(self, request):
-    # Check for a payload field. If found, this is a V1 service request.
-    payload_json = request.form.get('payload')
-    if payload_json:
-      return self.handle_V1_trigger_request(request)
-
-    # Otherwise, this is a V2 webhook request.
-    return self.handle_V2_trigger_request(request)
-
-  def handle_V2_trigger_request(self, request):
     payload = request.get_json()
     if not 'push' in payload:
       logger.debug('Skipping BitBucket request due to missing push data in payload')
@@ -607,48 +589,6 @@ class BitbucketBuildTrigger(BuildTriggerHandler):
                                actor=payload.get('actor'))
 
 
-  def handle_V1_trigger_request(self, request):
-    # TODO(jschorr): Delete this code after the migration has successfully completed.
-
-    # Parse the JSON payload.
-    payload_json = request.form.get('payload')
-    if not payload_json:
-      logger.debug('Skipping BitBucket request due to missing payload')
-      raise SkipRequestException()
-
-    try:
-      payload = json.loads(payload_json)
-    except ValueError:
-      logger.debug('Skipping BitBucket request due to invalid payload')
-      raise SkipRequestException()
-
-    logger.debug('BitBucket trigger payload %s', payload)
-
-    # Make sure we have a commit in the payload.
-    if not payload.get('commits'):
-      logger.debug('Skipping BitBucket request due to missing commits block')
-      raise SkipRequestException()
-
-    # Check if this build should be skipped by commit message.
-    commit = payload['commits'][-1]
-    commit_message = commit['message']
-    if should_skip_commit(commit_message):
-      logger.debug('Skipping BitBucket request due to commit message request')
-      raise SkipRequestException()
-
-    # Check to see if this build should be skipped by ref.
-    if not commit.get('branch') and not commit.get('tag'):
-      logger.debug('Skipping BitBucket request due to missing branch and tag')
-      raise SkipRequestException()
-
-    ref = 'refs/heads/' + commit['branch'] if commit.get('branch') else 'refs/tags/' + commit['tag']
-    logger.debug('Checking BitBucket request: %s', ref)
-    raise_if_skipped(self.config, ref)
-
-    commit_sha = commit['node']
-    return self._prepare_build(commit_sha, ref, False)
-
-
   def manual_start(self, run_parameters=None):
     run_parameters = run_parameters or {}
     repository = self._get_repository_client()

util/migratebitbucketservices.py

@@ -0,0 +1,90 @@
+import logging
+import json
+
+from app import app
+from data.database import configure, RepositoryBuildTrigger, BuildTriggerService
+from bitbucket import BitBucket
+from endpoints.trigger import BitbucketBuildTrigger
+
+configure(app.config)
+
+logger = logging.getLogger(__name__)
+
+def run_bitbucket_migration():
+  bitbucket_trigger = BuildTriggerService.get(BuildTriggerService.name == "bitbucket")
+
+  encountered = set()
+  while True:
+    found = list(RepositoryBuildTrigger.select().where(
+                      RepositoryBuildTrigger.service == bitbucket_trigger,
+                      RepositoryBuildTrigger.config ** "%\"hook_id%"))
+
+    found = [f for f in found if not f.uuid in encountered]
+
+    if not found:
+      logger.debug('No additional records found')
+      return
+
+    logger.debug('Found %s records to be changed', len(found))
+    for trigger in found:
+      encountered.add(trigger.uuid)
+
+      try:
+        config = json.loads(trigger.config)
+      except:
+        logging.error("Cannot parse config for trigger %s", trigger.uuid)
+        continue
+
+      logger.debug("Checking trigger %s", trigger.uuid)
+      if 'hook_id' in config:
+        logger.debug("Updating trigger %s to a webhook", trigger.uuid)
+
+        trigger_handler = BitbucketBuildTrigger(trigger)
+        client = trigger_handler._get_repository_client()
+
+        hook_id = config['hook_id']
+
+        # Lookup the old service hook.
+        logger.debug("Looking up old service URL for trigger %s", trigger.uuid)
+        (result, hook_data, err_msg) = client.services().get(hook_id)
+        if not result or not hook_data:
+          logger.error('Error when retrieving service hook for trigger %s: %s', trigger.uuid, err_msg)
+          continue
+
+        if not 'webhook_id' in config:
+          hook_data = hook_data[0]['service']
+          webhook_url = [f for f in hook_data['fields'] if f['name'] == 'URL'][0]['value']
+          logger.debug("Adding webhook for trigger %s: %s", trigger.uuid, webhook_url)
+
+          # Add the new web hook.
+          description = 'Webhook for invoking builds on %s' % app.config['REGISTRY_TITLE_SHORT']
+          webhook_events = ['repo:push']
+          (result, data, err_msg) = client.webhooks().create(description, webhook_url, webhook_events)
+          if not result:
+            logger.error('Error when adding webhook for trigger %s: %s', trigger.uuid, err_msg)
+            continue
+
+          config['webhook_id'] = data['uuid']
+          trigger.config = json.dumps(config)
+          trigger.save()
+
+        # Remove the old service hook.
+        logger.debug("Deleting  old service URL for trigger %s", trigger.uuid)
+        (result, _, err_msg) = client.services().delete(hook_id)
+        if not result:
+          logger.error('Error when deleting service hook for trigger %s: %s', trigger.uuid, err_msg)
+          continue
+
+        del config['hook_id']
+
+        # Update the config.
+        trigger.config = json.dumps(config)
+        trigger.save()
+        logger.debug("Trigger %s updated to a webhook", trigger.uuid)
+
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.DEBUG)
+  logging.getLogger('boto').setLevel(logging.CRITICAL)
+
+  run_bitbucket_migration()