diff --git a/buildserver/buildserver.py b/buildserver/buildserver.py index f9eba8416..6ebdd5c9b 100644 --- a/buildserver/buildserver.py +++ b/buildserver/buildserver.py @@ -4,9 +4,11 @@ import shutil import os import re -from flask import Flask, request, send_file, make_response, jsonify +from flask import Flask, request, send_file, jsonify, redirect, url_for, abort from zipfile import ZipFile from tempfile import TemporaryFile, mkdtemp +from uuid import uuid4 +from multiprocessing.pool import ThreadPool BUFFER_SIZE = 8 * 1024 @@ -17,97 +19,173 @@ app = Flask(__name__) logger = logging.getLogger(__name__) -def count_steps(dockerfileobj): - steps = 0 - for line in dockerfileobj.readlines(): - stripped = line.strip() - if stripped and stripped[0] is not '#': - steps += 1 - return steps - @app.route('/') def index(): return send_file('test.html') -@app.route('/start/zip/', methods=['POST']) -def start_build(tag_name): - docker_zip = request.files['dockerfile'] +def count_steps(dockerfile_path): + with open(dockerfile_path, 'r') as dockerfileobj: + steps = 0 + for line in dockerfileobj.readlines(): + stripped = line.strip() + if stripped and stripped[0] is not '#': + steps += 1 + return steps + +def prepare_zip(request_file): build_dir = mkdtemp(prefix='docker-build-') # Save the zip file to temp somewhere with TemporaryFile() as zip_file: - docker_zip.save(zip_file) + request_file.save(zip_file) to_extract = ZipFile(zip_file) to_extract.extractall(build_dir) - docker_cl = docker.Client(version='1.5') + return build_dir - build_status = docker_cl.build(path=build_dir, tag=tag_name) +def prepare_dockerfile(request_file): + build_dir = mkdtemp(prefix='docker-build-') dockerfile_path = os.path.join(build_dir, "Dockerfile") - with open(dockerfile_path, 'r') as dockerfileobj: - num_steps = count_steps(dockerfileobj) - logger.debug('Dockerfile had %s steps' % num_steps) + request_file.save(dockerfile_path) - current_step = 0 - built_image = None - for status in build_status: - step_increment = re.search(r'Step ([0-9]+) :', status) - if step_increment: - current_step = int(step_increment.group(1)) - logger.debug('Step now: %s/%s' % (current_step, num_steps)) - continue + return build_dir - complete = re.match(r'Successfully built ([a-z0-9]+)', status) - if complete: - built_image = complete.group(1) - logger.debug('Final image ID is: %s' % built_image) - continue - shutil.rmtree(build_dir) +MIME_PROCESSORS = { + 'application/zip': prepare_zip, + 'text/plain': prepare_dockerfile, +} - # Get the image count - if not built_image: - abort(500) - history = docker_cl.history(built_image) - num_images = len(history) +builds = {} +pool = ThreadPool(1) - logger.debug('Pushing to tag name: %s' % tag_name) - resp = docker_cl.push(tag_name) - current_image = 0 - image_progress = 0 - for status in resp: - if u'status' in status: - status_msg = status[u'status'] +def build_image(build_dir, tag_name, num_steps, result_object): + try: + logger.debug('Does this show up?') + docker_cl = docker.Client(version='1.5') + result_object['status'] = 'building' + build_status = docker_cl.build(path=build_dir, tag=tag_name) - next_image = r'(Pushing:|Image) [a-z0-9]+( already pushed, skipping)?' - match = re.match(next_image, status_msg) - if match: - current_image += 1 - image_progress = 0 - logger.debug('Now pushing image %s/%s' % (current_image, num_images)) + current_step = 0 + built_image = None + for status in build_status: + step_increment = re.search(r'Step ([0-9]+) :', status) + if step_increment: + current_step = int(step_increment.group(1)) + logger.debug('Step now: %s/%s' % (current_step, num_steps)) + result_object['current_command'] = current_step continue - if status_msg == u'Pushing' and u'progress' in status: - percent = r'\(([0-9]+)%\)' - match = re.search(percent, status[u'progress']) + complete = re.match(r'Successfully built ([a-z0-9]+)$', status) + if complete: + built_image = complete.group(1) + logger.debug('Final image ID is: %s' % built_image) + continue + + shutil.rmtree(build_dir) + + # Get the image count + if not built_image: + result_object['status'] = 'error' + result_object['message'] = 'Unable to build dockerfile.' + return + + history = docker_cl.history(built_image) + num_images = len(history) + result_object['total_images'] = num_images + + result_object['status'] = 'pushing' + logger.debug('Pushing to tag name: %s' % tag_name) + resp = docker_cl.push(tag_name) + + current_image = 0 + image_progress = 0 + for status in resp: + logger.debug(status) + + if u'status' in status: + status_msg = status[u'status'] + + next_image = r'(Pushing|Image) [a-z0-9]+( already pushed, skipping)?$' + match = re.match(next_image, status_msg) if match: - image_progress = int(match.group(1)) - continue + current_image += 1 + image_progress = 0 + logger.debug('Now pushing image %s/%s' % + (current_image, num_images)) - return jsonify({ - 'images_pushed': num_images, - 'commands_run': num_steps, - }) + elif status_msg == u'Pushing' and u'progress' in status: + percent = r'\(([0-9]+)%\)' + match = re.search(percent, status[u'progress']) + if match: + image_progress = int(match.group(1)) + + result_object['current_image'] = current_image + result_object['image_completion_percent'] = image_progress + + elif u'errorDetail' in status: + result_object['status'] = 'error' + if u'message' in status[u'errorDetail']: + result_object['message'] = status[u'errorDetail'][u'message'] + return + + result_object['status'] = 'complete' + except Exception as e: + logger.exception('Exception when processing request.') + result_object['status'] = 'error' + result_object['message'] = e.message -@app.route('/status') -def get_status(): - return 'building' +@app.route('/build', methods=['POST']) +def start_build(): + docker_input = request.files['dockerfile'] + c_type = docker_input.content_type + tag_name = request.values['tag'] + + logger.info('Request to build file of type: %s with tag: %s' % + (c_type, tag_name)) + + if c_type not in MIME_PROCESSORS: + logger.error('Invalid dockerfile content type: %s' % c_type) + abort(400) + + build_dir = MIME_PROCESSORS[c_type](docker_input) + + dockerfile_path = os.path.join(build_dir, "Dockerfile") + num_steps = count_steps(dockerfile_path) + logger.debug('Dockerfile had %s steps' % num_steps) + + job_id = str(uuid4()) + logger.info('Sending job to builder pool: %s' % job_id) + + result_object = { + 'id': job_id, + 'total_commands': num_steps, + 'total_images': None, + 'current_command': 0, + 'current_image': 0, + 'image_completion_percent': 0, + 'status': 'waiting', + 'message': None, + } + builds[job_id] = result_object + pool.apply_async(build_image, [build_dir, tag_name, num_steps, + result_object]) + + return redirect(url_for('get_status', job_id=job_id)) + + +@app.route('/status/') +def get_status(job_id): + if job_id not in builds: + abort(400) + + return jsonify(builds[job_id]) if __name__ == '__main__': diff --git a/buildserver/requirements-nover.txt b/buildserver/requirements-nover.txt index 7a7ebd121..e5ec39f3c 100644 --- a/buildserver/requirements-nover.txt +++ b/buildserver/requirements-nover.txt @@ -1,2 +1,2 @@ flask --e git+git://github.com/dotcloud/docker-py.git#egg=docker-py \ No newline at end of file +-e git+git://github.com/DevTable/docker-py.git#egg=docker-py diff --git a/buildserver/test.html b/buildserver/test.html index 472f4dd6d..0137b766b 100644 --- a/buildserver/test.html +++ b/buildserver/test.html @@ -1,7 +1,8 @@ -
+ +