From 7f17c6da24fc84ee6df44d06e8611779deaaf91a Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 14:42:16 -0400 Subject: [PATCH 1/9] Fix the forking problem in the buildserver. --- buildserver/buildserver.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 8ee7f4089..82cb24ac7 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -146,7 +146,15 @@ build = { pool = ThreadPool(1) -def start_build(resource_url, tag_name, acccess_token): +@app.before_first_request +def start_build(): + resource_url = os.environ['RESOURCE_URL'] + tag_name = os.environ['TAG'] + acccess_token = os.environ['TOKEN'] + + logger.debug('Starting job with resource url: %s tag: %s and token: %s' % + (resource_url, tag_name, acccess_token)) + # Save the token host = re.match(r'([a-z0-9.:]+)/.+/.+$', tag_name) if host: @@ -201,13 +209,4 @@ def health_check(): if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT) - resource_url = os.environ['RESOURCE_URL'] - tag_name = os.environ['TAG'] - acccess_token = os.environ['TOKEN'] - - logger.debug('Starting job with resource url: %s tag: %s and token: %s' % - (resource_url, tag_name, acccess_token)) - - start_build(resource_url, tag_name, acccess_token) - app.run(host='0.0.0.0', port=5002) From 798cf781728a786f26440070bed30109ad93ec6d Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 15:39:44 -0400 Subject: [PATCH 2/9] Re-add cleaning up the DO node. --- workers/dockerfilebuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index a1bd7ede3..dc4209d35 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -183,7 +183,7 @@ def babysit_builder(request): # clean up the DO node logger.debug('Cleaning up DO node.') - # retry_command(droplet.destroy) + retry_command(droplet.destroy) repository_build.status_url = None repository_build.build_node_id = None; From 6f105326aa931471941f9e0ea33672d8e7d7fa5e Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 15:42:19 -0400 Subject: [PATCH 3/9] Return unfinished items to the queue when they can be retried. --- data/queue.py | 4 ++++ workers/dockerfilebuild.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/data/queue.py b/data/queue.py index 3038b8389..8e63c4d17 100644 --- a/data/queue.py +++ b/data/queue.py @@ -54,6 +54,10 @@ class WorkQueue(object): def complete(self, completed_item): completed_item.delete_instance() + def incomplete(self, incomplete_item): + incomplete_item.available = True + incomplete_item.save() + image_diff_queue = WorkQueue('imagediff') dockerfile_build_queue = WorkQueue('dockerfilebuild') diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index dc4209d35..2f7aea2da 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -212,6 +212,10 @@ def process_work_items(pool): def complete_callback(completed): if completed: dockerfile_build_queue.complete(local_item) + else: + # We have a retryable error, add the job back to the queue + dockerfile_build_queue.incomplete(local_item) + return complete_callback logger.debug('Sending work item to thread pool: %s' % pool) From 78d6c9d9a38d86bf1a04c170c059df195401b8f8 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 16:11:54 -0400 Subject: [PATCH 4/9] Just use the current server as the registry server, this will work for all externally available addresses. --- config.py | 3 --- endpoints/index.py | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index 3647c886e..de81bdfe6 100644 --- a/config.py +++ b/config.py @@ -103,7 +103,6 @@ class BuildNodeConfig(object): class DebugConfig(FlaskConfig, MailConfig, LocalStorage, SQLiteDB, StripeTestConfig, MixpanelTestConfig, GitHubTestConfig, DigitalOceanConfig, AWSCredentials, BuildNodeConfig): - REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, 'format': LOG_FORMAT @@ -117,7 +116,6 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelTestConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig): - REGISTRY_SERVER = 'localhost:5000' LOGGING_CONFIG = { 'level': logging.DEBUG, 'format': LOG_FORMAT @@ -128,7 +126,6 @@ class LocalHostedConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, class ProductionConfig(FlaskConfig, MailConfig, S3Storage, RDSMySQL, StripeLiveConfig, MixpanelProdConfig, GitHubProdConfig, DigitalOceanConfig, BuildNodeConfig): - REGISTRY_SERVER = 'quay.io' LOGGING_CONFIG = { 'stream': sys.stderr, 'level': logging.DEBUG, diff --git a/endpoints/index.py b/endpoints/index.py index 375fa94b4..4920e2b15 100644 --- a/endpoints/index.py +++ b/endpoints/index.py @@ -2,6 +2,7 @@ import json import urllib import json import logging +import urlparse from flask import request, make_response, jsonify, abort from functools import wraps @@ -25,7 +26,9 @@ def generate_headers(role='read'): def wrapper(namespace, repository, *args, **kwargs): response = f(namespace, repository, *args, **kwargs) - response.headers['X-Docker-Endpoints'] = app.config['REGISTRY_SERVER'] + # We run our index and registry on the same hosts for now + registry_server = urlparse.urlparse(request.url).netloc + response.headers['X-Docker-Endpoints'] = registry_server has_token_request = request.headers.get('X-Docker-Token', '') From 6b6a238d3acc5fe83cdfd76f3cfec690f2b8fa8d Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:13:27 -0400 Subject: [PATCH 5/9] Not all hosts may have apt-add-repository. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2f608b81b..1e658d180 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ to prepare a new host: ``` +sudo apt-get install software-properties-common sudo apt-add-repository -y ppa:nginx/stable sudo apt-get update sudo apt-get install -y git python-virtualenv python-dev phantomjs From ba5553e9a816224bec832434d10a5b8ce9eb54a5 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:14:00 -0400 Subject: [PATCH 6/9] Make the nginx config ALMOST work on digital ocean. --- nginx.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nginx.conf b/nginx.conf index 93e5a1ce1..68b3ed5c5 100644 --- a/nginx.conf +++ b/nginx.conf @@ -10,6 +10,7 @@ events { } http { + types_hash_max_size 2048; include /etc/nginx/mime.types; default_type application/octet-stream; @@ -36,8 +37,8 @@ http { keepalive_timeout 5; ssl on; - ssl_certificate /home/ubuntu/quay/certs/unified.cert; - ssl_certificate_key /home/ubuntu/quay/certs/quay.key; + ssl_certificate ./certs/unified.cert; + ssl_certificate_key ./certs/quay.key; ssl_session_timeout 5m; ssl_protocols SSLv3 TLSv1; ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP; From d3d593f7e134f9bce41e92cbdba8f024c0b55168 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:14:22 -0400 Subject: [PATCH 7/9] Add some missing dependencies to for the workers. --- requirements-nover.txt | 3 ++- requirements.txt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements-nover.txt b/requirements-nover.txt index a12714454..8d95e3e48 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -15,4 +15,5 @@ beautifulsoup4 marisa-trie apscheduler python-daemon -paramiko \ No newline at end of file +paramiko +python-digitalocean \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 171930241..494ed9198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ beautifulsoup4==4.3.2 blinker==1.3 boto==2.15.0 distribute==0.6.34 +ecdsa==0.10 eventlet==0.14.0 greenlet==0.4.1 gunicorn==18.0 @@ -19,10 +20,13 @@ itsdangerous==0.23 lockfile==0.9.1 marisa-trie==0.5.1 mixpanel-py==3.0.0 +paramiko==1.12.0 peewee==2.1.4 py-bcrypt==0.4 +pycrypto==2.6.1 python-daemon==1.6 python-dateutil==2.1 +python-digitalocean==0.5 requests==2.0.0 six==1.4.1 stripe==1.9.8 From 5e81f999d1bab74d0fa24cd18ef4a8c4f6b22566 Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:15:12 -0400 Subject: [PATCH 8/9] Make the build server and build worker slightly more robust to errors. --- buildserver/buildserver.py | 2 +- workers/dockerfilebuild.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index 82cb24ac7..ebd039cdc 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -124,7 +124,7 @@ def build_image(build_dir, tag_name, num_steps, result_object): except Exception as e: logger.exception('Exception when processing request.') result_object['status'] = 'error' - result_object['message'] = e.message + result_object['message'] = str(e.message) MIME_PROCESSORS = { diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 2f7aea2da..5112145b0 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -43,7 +43,7 @@ def retry_command(to_call, args=[], kwargs={}, retries=5, period=5): def get_status(url): - return requests.get(url).json()['status'] + return retry_command(requests.get, [url]).json()['status'] def babysit_builder(request): @@ -112,8 +112,10 @@ def babysit_builder(request): ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + logger.debug('Connecting to droplet through ssh at ip: %s' % + droplet.ip_address) retry_command(ssh_client.connect, [droplet.ip_address, 22, 'root'], - {'look_for_keys': False, + {'look_for_keys': False, 'timeout': 10.0, 'key_filename': app.config['DO_SSH_PRIVATE_KEY_FILENAME']}) # Load the node with the pull token From 7adf37e6b57c43e3ac5fd6d1b24c7a290df24cfb Mon Sep 17 00:00:00 2001 From: yackob03 Date: Tue, 29 Oct 2013 18:56:57 -0400 Subject: [PATCH 9/9] Move the parmiko import to work around a bug with python daemonization and paramiko. --- workers/dockerfilebuild.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 5112145b0..7a3daa12f 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -5,7 +5,6 @@ import time import argparse import digitalocean import requests -import paramiko from apscheduler.scheduler import Scheduler from multiprocessing.pool import ThreadPool @@ -109,6 +108,9 @@ def babysit_builder(request): repository_build.phase = 'initializing' repository_build.save() + # We wait until here to import paramiko because otherwise it doesn't work + # under the daemon context. + import paramiko ssh_client = paramiko.SSHClient() ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())