vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Properly connect the github push webhook with the build worker. Still need to resolve the archive format.

Browse source
This commit is contained in:
jakedt 2014-02-18 18:09:14 -05:00
parent ed38bcdafc
commit f60f9eb62a
8 changed files with 57 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
config.py
View file

@ -1,5 +1,6 @@
import logging
import logstash_formatter
import requests
from peewee import MySQLDatabase, SqliteDatabase
from storage.s3 import S3Storage
@ -18,6 +19,7 @@ class FlaskConfig(object):
SECRET_KEY = '1cb18882-6d12-440d-a4cc-b7430fb5f884'
JSONIFY_PRETTYPRINT_REGULAR = False
class FlaskProdConfig(FlaskConfig):
SESSION_COOKIE_SECURE = True
@ -163,9 +165,22 @@ def logs_init_builder(level=logging.DEBUG,
return init_logs
def build_requests_session():
sess = requests.Session()
adapter = requests.adapters.HTTPAdapter(pool_connections=100,
pool_maxsize=100)
sess.mount('http://', adapter)
sess.mount('https://', adapter)
return sess
class LargePoolHttpClient(object):
HTTPCLIENT = build_requests_session()
class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
FakeAnalytics, StripeTestConfig, RedisBuildLogs,
UserEventConfig):
UserEventConfig, LargePoolHttpClient):
LOGGING_CONFIG = logs_init_builder(logging.WARN)
POPULATE_DB_TEST_DATA = True
TESTING = True
@ -174,7 +189,7 @@ class TestConfig(FlaskConfig, FakeStorage, EphemeralDB, FakeUserfiles,
class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB,
StripeTestConfig, MixpanelTestConfig, GitHubTestConfig,
DigitalOceanConfig, BuildNodeConfig, S3Userfiles,
UserEventConfig, TestBuildLogs):
UserEventConfig, TestBuildLogs, LargePoolHttpClient):
LOGGING_CONFIG = logs_init_builder(formatter=logging.Formatter())
SEND_FILE_MAX_AGE_DEFAULT = 0
POPULATE_DB_TEST_DATA = True
@ -184,7 +199,7 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,
StripeLiveConfig, MixpanelTestConfig,
GitHubProdConfig, DigitalOceanConfig,
BuildNodeConfig, S3Userfiles, RedisBuildLogs,
UserEventConfig):
UserEventConfig, LargePoolHttpClient):
LOGGING_CONFIG = logs_init_builder()
SEND_FILE_MAX_AGE_DEFAULT = 0
@ -192,7 +207,8 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL,
class ProductionConfig(FlaskProdConfig, MailConfig, S3Storage, RDSMySQL,
StripeLiveConfig, MixpanelProdConfig,
GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig,
S3Userfiles, RedisBuildLogs, UserEventConfig):
S3Userfiles, RedisBuildLogs, UserEventConfig,
LargePoolHttpClient):
LOGGING_CONFIG = logs_init_builder()
SEND_FILE_MAX_AGE_DEFAULT = 0

9
data/userfiles.py
View file

@ -40,14 +40,15 @@ class UserRequestFiles(object):
encrypt_key=True)
return (url, file_id)
def store_file(self, flask_file):
def store_file(self, file_like_obj, content_type):
self._initialize_s3()
file_id = str(uuid4())
full_key = os.path.join(self._prefix, file_id)
k = Key(self._bucket, full_key)
logger.debug('Setting s3 content type to: %s' % flask_file.content_type)
k.set_metadata('Content-Type', flask_file.content_type)
bytes_written = k.set_contents_from_file(flask_file, encrypt_key=True)
logger.debug('Setting s3 content type to: %s' % content_type)
k.set_metadata('Content-Type', content_type)
bytes_written = k.set_contents_from_file(file_like_obj, encrypt_key=True,
rewind=True)
if bytes_written == 0:
raise S3FileWriteException('Unable to write file to S3')

7
endpoints/api.py
View file

@ -1,10 +1,10 @@
import logging
import stripe
import requests
import urlparse
import json
from flask import request, make_response, jsonify, abort, url_for, Blueprint, session
from flask import (request, make_response, jsonify, abort, url_for, Blueprint,
session)
from flask.ext.login import current_user, logout_user
from flask.ext.principal import identity_changed, AnonymousIdentity
from functools import wraps
@ -14,7 +14,8 @@ from data import model
from data.queue import dockerfile_build_queue
from data.plans import PLANS, get_plan
from app import app
from util.email import send_confirmation_email, send_recovery_email, send_change_email
from util.email import (send_confirmation_email, send_recovery_email,
send_change_email)
from util.names import parse_repository_name, format_robot_username
from util.gravatar import compute_hash

15
endpoints/callbacks.py
View file

@ -1,4 +1,3 @@
import requests
import logging
from flask import request, redirect, url_for, Blueprint
@ -12,9 +11,13 @@ from util.names import parse_repository_name
logger = logging.getLogger(__name__)
client = app.config['HTTPCLIENT']
callback = Blueprint('callback', __name__)
def exchange_github_code_for_token(code):
code = request.args.get('code')
payload = {
@ -26,8 +29,8 @@ def exchange_github_code_for_token(code):
'Accept': 'application/json'
}
get_access_token = requests.post(app.config['GITHUB_TOKEN_URL'],
params=payload, headers=headers)
get_access_token = client.post(app.config['GITHUB_TOKEN_URL'],
params=payload, headers=headers)
token = get_access_token.json()['access_token']
return token
@ -37,7 +40,7 @@ def get_github_user(token):
token_param = {
'access_token': token,
}
get_user = requests.get(app.config['GITHUB_USER_URL'], params=token_param)
get_user = client.get(app.config['GITHUB_USER_URL'], params=token_param)
return get_user.json()
@ -61,8 +64,8 @@ def github_oauth_callback():
token_param = {
'access_token': token,
}
get_email = requests.get(app.config['GITHUB_USER_EMAILS'],
params=token_param, headers=v3_media_type)
get_email = client.get(app.config['GITHUB_USER_EMAILS'], params=token_param,
headers=v3_media_type)
# We will accept any email, but we prefer the primary
found_email = None

28
endpoints/trigger.py
View file

@ -1,19 +1,21 @@
import json
import requests
import logging
import io
from github import Github
from tempfile import SpooledTemporaryFile
from app import app
user_files = app.config['USERFILES']
client = app.config['HTTPCLIENT']
logger = logging.getLogger(__name__)
ZIPBALL = 'application/zip'
CHUNK_SIZE = 512 * 1024
class BuildArchiveException(Exception):
@ -35,7 +37,7 @@ class BuildTrigger(object):
"""
raise NotImplementedError
def incoming_webhook(self, request, auth_token, config):
def handle_trigger_request(self, request, auth_token, config):
"""
Transform the incoming request data into a set of actions.
"""
@ -57,6 +59,10 @@ class BuildTrigger(object):
raise InvalidServiceException('Unable to find service: %s' % service)
def raise_unsupported():
raise io.UnsupportedOperation
class GithubBuildTrigger(BuildTrigger):
@staticmethod
def _get_client(auth_token):
@ -77,7 +83,7 @@ class GithubBuildTrigger(BuildTrigger):
return repo_list
def incoming_webhook(self, request, auth_token, config):
def handle_trigger_request(self, request, auth_token, config):
payload = request.get_json()
logger.debug('Payload %s', payload)
ref = payload['ref']
@ -94,17 +100,13 @@ class GithubBuildTrigger(BuildTrigger):
# Prepare the download and upload URLs
branch_name = ref.split('/')[-1]
archive_link = repo.get_archive_link('zipball', branch_name)
download_archive = requests.get(archive_link, stream=True)
download_archive = client.get(archive_link, stream=True)
upload_url, dockerfile_id = user_files.prepare_for_drop(ZIPBALL)
up_headers = {'Content-Type': ZIPBALL}
upload_archive = requests.put(upload_url, headers=up_headers,
data=download_archive.raw)
with SpooledTemporaryFile(CHUNK_SIZE) as zipball:
for chunk in download_archive.iter_content(CHUNK_SIZE):
zipball.write(chunk)
if upload_archive.status_code / 100 != 2:
logger.debug('Failed to upload archive to s3')
raise BuildArchiveException('Unable to copy archie to s3 for ref: %s' %
ref)
dockerfile_id = user_files.store_file(zipball, ZIPBALL)
logger.debug('Successfully prepared job')

1
endpoints/web.py
View file

@ -1,5 +1,4 @@
import logging
import requests
import stripe
from flask import (abort, redirect, request, url_for, make_response, Response,

5
endpoints/webhooks.py
View file

@ -61,8 +61,9 @@ def github_push_webhook(namespace, repository, trigger_uuid):
handler = BuildTrigger.get_trigger_for_service(trigger.service.name)
logger.debug('Passing webhook request to handler %s', handler)
df_id, tag, name = handler.incoming_webhook(request, trigger.auth_token,
trigger.config)
df_id, tag, name = handler.handle_trigger_request(request,
trigger.auth_token,
trigger.config)
host = urlparse.urlparse(request.url).netloc
full_tag = '%s/%s/%s:%s' % (host, trigger.repository.namespace,

2
test/teststorage.py
View file

@ -30,7 +30,7 @@ class FakeUserfiles(object):
def prepare_for_drop(self, mime_type):
return ('http://fake/url', uuid4())
def store_file(self, flask_file):
def store_file(self, file_like_obj, content_type):
raise NotImplementedError()
def get_file_url(self, file_id, expires_in=300):