From f60f9eb62a936931193f551c8bd7d5f46da21b2f Mon Sep 17 00:00:00 2001 From: jakedt Date: Tue, 18 Feb 2014 18:09:14 -0500 Subject: [PATCH] Properly connect the github push webhook with the build worker. Still need to resolve the archive format. --- config.py | 24 ++++++++++++++++++++---- data/userfiles.py | 9 +++++---- endpoints/api.py | 7 ++++--- endpoints/callbacks.py | 15 +++++++++------ endpoints/trigger.py | 28 +++++++++++++++------------- endpoints/web.py | 1 - endpoints/webhooks.py | 5 +++-- test/teststorage.py | 2 +- 8 files changed, 57 insertions(+), 34 deletions(-) diff --git a/config.py b/config.py index c81667d4d..bfb36426f 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,6 @@ import logging import logstash_formatter +import requests from peewee import MySQLDatabase, SqliteDatabase from storage.s3 import S3Storage @@ -18,6 +19,7 @@ class FlaskConfig(object): SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884' JSONIFY_PRETTYPRINT_REGULAR = False + class FlaskProdConfig(FlaskConfig): SESSION_COOKIE_SECURE = True @@ -163,9 +165,22 @@ def logs_init_builder(level=logging.DEBUG, return init_logs +def build_requests_session(): + sess = requests.Session() + adapter = requests.adapters.HTTPAdapter(pool_connections=100, + pool_maxsize=100) + sess.mount('http://', adapter) + sess.mount('https://', adapter) + return sess + + +class LargePoolHttpClient(object): + HTTPCLIENT = build_requests_session() + + class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, FakeAnalytics, StripeTestConfig, RedisBuildLogs, - UserEventConfig): + UserEventConfig, LargePoolHttpClient): LOGGING_CONFIG = logs_init_builder(logging.WARN) POPULATE_DB_TEST_DATA = True TESTING = True @@ -174,7 +189,7 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles, class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles, - UserEventConfig, TestBuildLogs): + UserEventConfig, TestBuildLogs, LargePoolHttpClient): LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter()) SEND_FILE_MAX_AGE_DEFAULT = 0 POPULATE_DB_TEST_DATA = True @@ -184,7 +199,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, S3Userfiles, RedisBuildLogs, - UserEventConfig): + UserEventConfig, LargePoolHttpClient): LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 @@ -192,7 +207,8 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig, - S3Userfiles, RedisBuildLogs, UserEventConfig): + S3Userfiles, RedisBuildLogs, UserEventConfig, + LargePoolHttpClient): LOGGING_CONFIG = logs_init_builder() SEND_FILE_MAX_AGE_DEFAULT = 0 diff --git a/data/userfiles.py b/data/userfiles.py index c2a8bc63c..cc314a47f 100644 --- a/data/userfiles.py +++ b/data/userfiles.py @@ -40,14 +40,15 @@ class UserRequestFiles(object): encrypt_key=True) return (url, file_id) - def store_file(self, flask_file): + def store_file(self, file_like_obj, content_type): self._initialize_s3() file_id = str(uuid4()) full_key = os.path.join(self._prefix, file_id) k = Key(self._bucket, full_key) - logger.debug('Setting s3 content type to: %s' % flask_file.content_type) - k.set_metadata('Content-Type', flask_file.content_type) - bytes_written = k.set_contents_from_file(flask_file, encrypt_key=True) + logger.debug('Setting s3 content type to: %s' % content_type) + k.set_metadata('Content-Type', content_type) + bytes_written = k.set_contents_from_file(file_like_obj, encrypt_key=True, + rewind=True) if bytes_written == 0: raise S3FileWriteException('Unable to write file to S3') diff --git a/endpoints/api.py b/endpoints/api.py index 389c74e0c..4b307337d 100644 --- a/endpoints/api.py +++ b/endpoints/api.py @@ -1,10 +1,10 @@ import logging import stripe -import requests import urlparse import json -from flask import request, make_response, jsonify, abort, url_for, Blueprint, session +from flask import (request, make_response, jsonify, abort, url_for, Blueprint, + session) from flask.ext.login import current_user, logout_user from flask.ext.principal import identity_changed, AnonymousIdentity from functools import wraps @@ -14,7 +14,8 @@ from data import model from data.queue import dockerfile_build_queue from data.plans import PLANS, get_plan from app import app -from util.email import send_confirmation_email, send_recovery_email, send_change_email +from util.email import (send_confirmation_email, send_recovery_email, + send_change_email) from util.names import parse_repository_name, format_robot_username from util.gravatar import compute_hash diff --git a/endpoints/callbacks.py b/endpoints/callbacks.py index 26b35f456..e78bda55e 100644 --- a/endpoints/callbacks.py +++ b/endpoints/callbacks.py @@ -1,4 +1,3 @@ -import requests import logging from flask import request, redirect, url_for, Blueprint @@ -12,9 +11,13 @@ from util.names import parse_repository_name logger = logging.getLogger(__name__) +client = app.config['HTTPCLIENT'] + + callback = Blueprint('callback', __name__) + def exchange_github_code_for_token(code): code = request.args.get('code') payload = { @@ -26,8 +29,8 @@ def exchange_github_code_for_token(code): 'Accept': 'application/json' } - get_access_token = requests.post(app.config['GITHUB_TOKEN_URL'], - params=payload, headers=headers) + get_access_token = client.post(app.config['GITHUB_TOKEN_URL'], + params=payload, headers=headers) token = get_access_token.json()['access_token'] return token @@ -37,7 +40,7 @@ def get_github_user(token): token_param = { 'access_token': token, } - get_user = requests.get(app.config['GITHUB_USER_URL'], params=token_param) + get_user = client.get(app.config['GITHUB_USER_URL'], params=token_param) return get_user.json() @@ -61,8 +64,8 @@ def github_oauth_callback(): token_param = { 'access_token': token, } - get_email = requests.get(app.config['GITHUB_USER_EMAILS'], - params=token_param, headers=v3_media_type) + get_email = client.get(app.config['GITHUB_USER_EMAILS'], params=token_param, + headers=v3_media_type) # We will accept any email, but we prefer the primary found_email = None diff --git a/endpoints/trigger.py b/endpoints/trigger.py index 37d8fbf33..8242e7ee6 100644 --- a/endpoints/trigger.py +++ b/endpoints/trigger.py @@ -1,19 +1,21 @@ -import json -import requests import logging +import io from github import Github +from tempfile import SpooledTemporaryFile from app import app user_files = app.config['USERFILES'] +client = app.config['HTTPCLIENT'] logger = logging.getLogger(__name__) ZIPBALL = 'application/zip' +CHUNK_SIZE = 512 * 1024 class BuildArchiveException(Exception): @@ -35,7 +37,7 @@ class BuildTrigger(object): """ raise NotImplementedError - def incoming_webhook(self, request, auth_token, config): + def handle_trigger_request(self, request, auth_token, config): """ Transform the incoming request data into a set of actions. """ @@ -57,6 +59,10 @@ class BuildTrigger(object): raise InvalidServiceException('Unable to find service: %s' % service) +def raise_unsupported(): + raise io.UnsupportedOperation + + class GithubBuildTrigger(BuildTrigger): @staticmethod def _get_client(auth_token): @@ -77,7 +83,7 @@ class GithubBuildTrigger(BuildTrigger): return repo_list - def incoming_webhook(self, request, auth_token, config): + def handle_trigger_request(self, request, auth_token, config): payload = request.get_json() logger.debug('Payload %s', payload) ref = payload['ref'] @@ -94,17 +100,13 @@ class GithubBuildTrigger(BuildTrigger): # Prepare the download and upload URLs branch_name = ref.split('/')[-1] archive_link = repo.get_archive_link('zipball', branch_name) - download_archive = requests.get(archive_link, stream=True) + download_archive = client.get(archive_link, stream=True) - upload_url, dockerfile_id = user_files.prepare_for_drop(ZIPBALL) - up_headers = {'Content-Type': ZIPBALL} - upload_archive = requests.put(upload_url, headers=up_headers, - data=download_archive.raw) + with SpooledTemporaryFile(CHUNK_SIZE) as zipball: + for chunk in download_archive.iter_content(CHUNK_SIZE): + zipball.write(chunk) - if upload_archive.status_code / 100 != 2: - logger.debug('Failed to upload archive to s3') - raise BuildArchiveException('Unable to copy archie to s3 for ref: %s' % - ref) + dockerfile_id = user_files.store_file(zipball, ZIPBALL) logger.debug('Successfully prepared job') diff --git a/endpoints/web.py b/endpoints/web.py index 3aa3617e1..9d898fbf7 100644 --- a/endpoints/web.py +++ b/endpoints/web.py @@ -1,5 +1,4 @@ import logging -import requests import stripe from flask import (abort, redirect, request, url_for, make_response, Response, diff --git a/endpoints/webhooks.py b/endpoints/webhooks.py index a35708e03..f153e633d 100644 --- a/endpoints/webhooks.py +++ b/endpoints/webhooks.py @@ -61,8 +61,9 @@ def github_push_webhook(namespace, repository, trigger_uuid): handler = BuildTrigger.get_trigger_for_service(trigger.service.name) logger.debug('Passing webhook request to handler %s', handler) - df_id, tag, name = handler.incoming_webhook(request, trigger.auth_token, - trigger.config) + df_id, tag, name = handler.handle_trigger_request(request, + trigger.auth_token, + trigger.config) host = urlparse.urlparse(request.url).netloc full_tag = '%s/%s/%s:%s' % (host, trigger.repository.namespace, diff --git a/test/teststorage.py b/test/teststorage.py index 41768e09d..51d1fc8eb 100644 --- a/test/teststorage.py +++ b/test/teststorage.py @@ -30,7 +30,7 @@ class FakeUserfiles(object): def prepare_for_drop(self, mime_type): return ('http://fake/url', uuid4()) - def store_file(self, flask_file): + def store_file(self, file_like_obj, content_type): raise NotImplementedError() def get_file_url(self, file_id, expires_in=300):