Move the source dockerfile processing to a modular system. Move actual build to a thread. Wire up the status API. Fix a bug with the way current_image was computed. Add untested support for raw dockerfile upload.

buildserver/buildserver.py

@@ -4,9 +4,11 @@ import shutil
 import os
 import re
 
-from flask import Flask, request, send_file, make_response, jsonify
+from flask import Flask, request, send_file, jsonify, redirect, url_for, abort
 from zipfile import ZipFile
 from tempfile import TemporaryFile, mkdtemp
+from uuid import uuid4
+from multiprocessing.pool import ThreadPool
 
 
 BUFFER_SIZE = 8 * 1024
@@ -17,7 +19,13 @@ app = Flask(__name__)
 logger = logging.getLogger(__name__)
 
 
-def count_steps(dockerfileobj):
+@app.route('/')
+def index():
+  return send_file('test.html')
+
+
+def count_steps(dockerfile_path):
+  with open(dockerfile_path, 'r') as dockerfileobj:
     steps = 0
     for line in dockerfileobj.readlines():
       stripped = line.strip()
@@ -25,31 +33,43 @@ def count_steps(dockerfileobj):
         steps += 1
     return steps
 
-@app.route('/')
-def index():
-  return send_file('test.html')
-
-
-@app.route('/start/zip/<path:tag_name>', methods=['POST'])
-def start_build(tag_name):
-  docker_zip = request.files['dockerfile']
 
+def prepare_zip(request_file):
   build_dir = mkdtemp(prefix='docker-build-')
 
   # Save the zip file to temp somewhere
   with TemporaryFile() as zip_file:
-    docker_zip.save(zip_file)
+    request_file.save(zip_file)
     to_extract = ZipFile(zip_file)
     to_extract.extractall(build_dir)
 
-  docker_cl = docker.Client(version='1.5')
+  return build_dir
 
-  build_status = docker_cl.build(path=build_dir, tag=tag_name)
 
+def prepare_dockerfile(request_file):
+  build_dir = mkdtemp(prefix='docker-build-')
   dockerfile_path = os.path.join(build_dir, "Dockerfile")
-  with open(dockerfile_path, 'r') as dockerfileobj:
+  request_file.save(dockerfile_path)
-    num_steps = count_steps(dockerfileobj)
+
-    logger.debug('Dockerfile had %s steps' % num_steps)
+  return build_dir
+
+
+MIME_PROCESSORS = {
+  'application/zip': prepare_zip,
+  'text/plain': prepare_dockerfile,
+}
+
+
+builds = {}
+pool = ThreadPool(1)
+
+
+def build_image(build_dir, tag_name, num_steps, result_object):
+  try:
+    logger.debug('Does this show up?')
+    docker_cl = docker.Client(version='1.5')
+    result_object['status'] = 'building'
+    build_status = docker_cl.build(path=build_dir, tag=tag_name)
 
     current_step = 0
     built_image = None
@@ -58,9 +78,10 @@ def start_build(tag_name):
       if step_increment:
         current_step = int(step_increment.group(1))
         logger.debug('Step now: %s/%s' % (current_step, num_steps))
+        result_object['current_command'] = current_step
         continue
 
-    complete = re.match(r'Successfully built ([a-z0-9]+)', status)
+      complete = re.match(r'Successfully built ([a-z0-9]+)$', status)
       if complete:
         built_image = complete.group(1)
         logger.debug('Final image ID is: %s' % built_image)
@@ -70,44 +91,101 @@ def start_build(tag_name):
 
     # Get the image count
     if not built_image:
-    abort(500)
+      result_object['status'] = 'error'
+      result_object['message'] = 'Unable to build dockerfile.'
+      return
 
     history = docker_cl.history(built_image)
     num_images = len(history)
+    result_object['total_images'] = num_images
 
+    result_object['status'] = 'pushing'
     logger.debug('Pushing to tag name: %s' % tag_name)
     resp = docker_cl.push(tag_name)
 
     current_image = 0
     image_progress = 0
     for status in resp:
+      logger.debug(status)
+
       if u'status' in status:
         status_msg = status[u'status']
 
-      next_image = r'(Pushing:|Image) [a-z0-9]+( already pushed, skipping)?'
+        next_image = r'(Pushing|Image) [a-z0-9]+( already pushed, skipping)?$'
         match = re.match(next_image, status_msg)
         if match:
           current_image += 1
           image_progress = 0
-        logger.debug('Now pushing image %s/%s' % (current_image, num_images))
+          logger.debug('Now pushing image %s/%s' %
-        continue
+                       (current_image, num_images))
 
-      if status_msg == u'Pushing' and u'progress' in status:
+        elif status_msg == u'Pushing' and u'progress' in status:
           percent = r'\(([0-9]+)%\)'
           match = re.search(percent, status[u'progress'])
           if match:
             image_progress = int(match.group(1))
-          continue
 
-  return jsonify({
+        result_object['current_image'] = current_image
-    'images_pushed': num_images,
+        result_object['image_completion_percent'] = image_progress
-    'commands_run': num_steps,
+
-  })
+      elif u'errorDetail' in status:
+        result_object['status'] = 'error'
+        if u'message' in status[u'errorDetail']:
+          result_object['message'] = status[u'errorDetail'][u'message']
+        return
+
+    result_object['status'] = 'complete'
+  except Exception as e:
+    logger.exception('Exception when processing request.')
+    result_object['status'] = 'error'
+    result_object['message'] = e.message
 
 
-@app.route('/status')
+@app.route('/build', methods=['POST'])
-def get_status():
+def start_build():
-  return 'building'
+  docker_input = request.files['dockerfile']
+  c_type = docker_input.content_type
+  tag_name = request.values['tag']
+
+  logger.info('Request to build file of type: %s with tag: %s' %
+              (c_type, tag_name))
+
+  if c_type not in MIME_PROCESSORS:
+    logger.error('Invalid dockerfile content type: %s' % c_type)
+    abort(400)
+
+  build_dir = MIME_PROCESSORS[c_type](docker_input)
+
+  dockerfile_path = os.path.join(build_dir, "Dockerfile")
+  num_steps = count_steps(dockerfile_path)
+  logger.debug('Dockerfile had %s steps' % num_steps)
+
+  job_id = str(uuid4())
+  logger.info('Sending job to builder pool: %s' % job_id)
+
+  result_object = {
+    'id': job_id,
+    'total_commands': num_steps,
+    'total_images': None,
+    'current_command': 0,
+    'current_image': 0,
+    'image_completion_percent': 0,
+    'status': 'waiting',
+    'message': None,
+  }
+  builds[job_id] = result_object
+  pool.apply_async(build_image, [build_dir, tag_name, num_steps,
+                                 result_object])
+
+  return redirect(url_for('get_status', job_id=job_id))
+
+
+@app.route('/status/<job_id>')
+def get_status(job_id):
+  if job_id not in builds:
+    abort(400)
+
+  return jsonify(builds[job_id])
 
 
 if __name__ == '__main__':

buildserver/requirements-nover.txt

@@ -1,2 +1,2 @@
 flask
--e git+git://github.com/dotcloud/docker-py.git#egg=docker-py
+-e git+git://github.com/DevTable/docker-py.git#egg=docker-py

buildserver/test.html

@@ -1,7 +1,8 @@
 <html>
   <body>
-    <form enctype='multipart/form-data' method="post" action="/start/zip/localhost:5000/devtable/newtag">
+    <form enctype='multipart/form-data' method="post" action="/build">
       <input type="file" name="dockerfile">
+      <input type="text" name="tag">
       <button type="submit">Send</button>
     </form>
   </body>