Properly connect the github push webhook with the build worker. Still need to resolve the archive format.

endpoints/api.py

@@ -1,10 +1,10 @@
 import logging
 import stripe
-import requests
 import urlparse
 import json
 
-from flask import request, make_response, jsonify, abort, url_for, Blueprint, session
+from flask import (request, make_response, jsonify, abort, url_for, Blueprint,
+                   session)
 from flask.ext.login import current_user, logout_user
 from flask.ext.principal import identity_changed, AnonymousIdentity
 from functools import wraps
@@ -14,7 +14,8 @@ from data import model
 from data.queue import dockerfile_build_queue
 from data.plans import PLANS, get_plan
 from app import app
-from util.email import send_confirmation_email, send_recovery_email, send_change_email
+from util.email import (send_confirmation_email, send_recovery_email,
+                        send_change_email)
 from util.names import parse_repository_name, format_robot_username
 from util.gravatar import compute_hash
 

endpoints/callbacks.py

@@ -1,4 +1,3 @@
-import requests
 import logging
 
 from flask import request, redirect, url_for, Blueprint
@@ -12,9 +11,13 @@ from util.names import parse_repository_name
 
 logger = logging.getLogger(__name__)
 
+client = app.config['HTTPCLIENT']
+
+
 callback = Blueprint('callback', __name__)
 
 
+
 def exchange_github_code_for_token(code):
   code = request.args.get('code')
   payload = {
@@ -26,8 +29,8 @@ def exchange_github_code_for_token(code):
     'Accept': 'application/json'
   }
 
-  get_access_token = requests.post(app.config['GITHUB_TOKEN_URL'],
-                                   params=payload, headers=headers)
+  get_access_token = client.post(app.config['GITHUB_TOKEN_URL'],
+                                 params=payload, headers=headers)
 
   token = get_access_token.json()['access_token']
   return token
@@ -37,7 +40,7 @@ def get_github_user(token):
   token_param = {
     'access_token': token,
   }
-  get_user = requests.get(app.config['GITHUB_USER_URL'], params=token_param)
+  get_user = client.get(app.config['GITHUB_USER_URL'], params=token_param)
 
   return get_user.json()
 
@@ -61,8 +64,8 @@ def github_oauth_callback():
   token_param = {
     'access_token': token,
   }
-  get_email = requests.get(app.config['GITHUB_USER_EMAILS'],
-                           params=token_param, headers=v3_media_type)
+  get_email = client.get(app.config['GITHUB_USER_EMAILS'], params=token_param,
+                         headers=v3_media_type)
 
   # We will accept any email, but we prefer the primary
   found_email = None

endpoints/trigger.py

@@ -1,19 +1,21 @@
-import json
-import requests
 import logging
+import io
 
 from github import Github
+from tempfile import SpooledTemporaryFile
 
 from app import app
 
 
 user_files = app.config['USERFILES']
+client = app.config['HTTPCLIENT']
 
 
 logger = logging.getLogger(__name__)
 
 
 ZIPBALL = 'application/zip'
+CHUNK_SIZE = 512 * 1024
 
 
 class BuildArchiveException(Exception):
@@ -35,7 +37,7 @@ class BuildTrigger(object):
     """
     raise NotImplementedError
 
-  def incoming_webhook(self, request, auth_token, config):
+  def handle_trigger_request(self, request, auth_token, config):
     """
     Transform the incoming request data into a set of actions.
     """
@@ -57,6 +59,10 @@ class BuildTrigger(object):
     raise InvalidServiceException('Unable to find service: %s' % service)
 
 
+def raise_unsupported():
+  raise io.UnsupportedOperation
+
+
 class GithubBuildTrigger(BuildTrigger):
   @staticmethod
   def _get_client(auth_token):
@@ -77,7 +83,7 @@ class GithubBuildTrigger(BuildTrigger):
 
     return repo_list
 
-  def incoming_webhook(self, request, auth_token, config):
+  def handle_trigger_request(self, request, auth_token, config):
     payload = request.get_json()
     logger.debug('Payload %s', payload)
     ref = payload['ref']
@@ -94,17 +100,13 @@ class GithubBuildTrigger(BuildTrigger):
     # Prepare the download and upload URLs
     branch_name = ref.split('/')[-1]
     archive_link = repo.get_archive_link('zipball', branch_name)
-    download_archive = requests.get(archive_link, stream=True)
+    download_archive = client.get(archive_link, stream=True)
 
-    upload_url, dockerfile_id = user_files.prepare_for_drop(ZIPBALL)
-    up_headers = {'Content-Type': ZIPBALL}
-    upload_archive = requests.put(upload_url, headers=up_headers,
-                                  data=download_archive.raw)
+    with SpooledTemporaryFile(CHUNK_SIZE) as zipball:
+      for chunk in download_archive.iter_content(CHUNK_SIZE):
+        zipball.write(chunk)
 
-    if upload_archive.status_code / 100 != 2:
-      logger.debug('Failed to upload archive to s3')
-      raise BuildArchiveException('Unable to copy archie to s3 for ref: %s' %
-                                  ref)
+      dockerfile_id = user_files.store_file(zipball, ZIPBALL)
 
     logger.debug('Successfully prepared job')
 

endpoints/web.py

@@ -1,5 +1,4 @@
 import logging
-import requests
 import stripe
 
 from flask import (abort, redirect, request, url_for, make_response, Response,

endpoints/webhooks.py

@@ -61,8 +61,9 @@ def github_push_webhook(namespace, repository, trigger_uuid):
     handler = BuildTrigger.get_trigger_for_service(trigger.service.name)
 
     logger.debug('Passing webhook request to handler %s', handler)
-    df_id, tag, name = handler.incoming_webhook(request, trigger.auth_token,
-                                                trigger.config)
+    df_id, tag, name = handler.handle_trigger_request(request,
+                                                      trigger.auth_token,
+                                                      trigger.config)
 
     host = urlparse.urlparse(request.url).netloc
     full_tag = '%s/%s/%s:%s' % (host, trigger.repository.namespace,