Fixes to the build server.

buildserver/Dockerfile

@@ -22,5 +22,5 @@ RUN venv/bin/pip install -r requirements.txt
 
 VOLUME /var/lib/docker
 
-EXPOSE 5002:5002
+EXPOSE 5002
 CMD startserver

buildserver/buildserver.py

@@ -6,10 +6,9 @@ import re
 import requests
 import json
 
-from flask import Flask, jsonify, url_for, abort, make_response
+from flask import Flask, jsonify, abort, make_response
 from zipfile import ZipFile
 from tempfile import TemporaryFile, mkdtemp
-from uuid import uuid4
 from multiprocessing.pool import ThreadPool
 from base64 import b64encode
 
@@ -53,16 +52,24 @@ def prepare_dockerfile(request_file):
   return build_dir
 
 
+def total_completion(statuses, total_images):
+  percentage_with_sizes = float(len(statuses.values()))/total_images
+  sent_bytes = sum([status[u'current'] for status in statuses.values()])
+  total_bytes = sum([status[u'total'] for status in statuses.values()])
+  return float(sent_bytes)/total_bytes*percentage_with_sizes
+
+
 def build_image(build_dir, tag_name, num_steps, result_object):
   try:
     logger.debug('Starting build.')
-    docker_cl = docker.Client(version='1.5')
+    docker_cl = docker.Client(timeout=1200)
     result_object['status'] = 'building'
-    build_status = docker_cl.build(path=build_dir, tag=tag_name)
+    build_status = docker_cl.build(path=build_dir, tag=tag_name, stream=True)
 
     current_step = 0
     built_image = None
     for status in build_status:
+      # logger.debug('Status: %s', str(status))
       step_increment = re.search(r'Step ([0-9]+) :', status)
       if step_increment:
         current_step = int(step_increment.group(1))
@@ -84,40 +91,36 @@ def build_image(build_dir, tag_name, num_steps, result_object):
       result_object['message'] = 'Unable to build dockerfile.'
       return
 
-    history = docker_cl.history(built_image)
+    history = json.loads(docker_cl.history(built_image))
     num_images = len(history)
     result_object['total_images'] = num_images
 
     result_object['status'] = 'pushing'
     logger.debug('Pushing to tag name: %s' % tag_name)
-    resp = docker_cl.push(tag_name)
+    resp = docker_cl.push(tag_name, stream=True)
 
-    current_image = 0
-    image_progress = 0
-    for status in resp:
+    for status_str in resp:
+      status = json.loads(status_str)
+      logger.debug('Status: %s', status_str)
       if u'status' in status:
         status_msg = status[u'status']
-        next_image = r'(Pushing|Image) [a-z0-9]+( already pushed, skipping)?$'
-        match = re.match(next_image, status_msg)
-        if match:
-          current_image += 1
-          image_progress = 0
-          logger.debug('Now pushing image %s/%s' %
-                       (current_image, num_images))
 
-        elif status_msg == u'Pushing' and u'progress' in status:
-          percent = r'\(([0-9]+)%\)'
-          match = re.search(percent, status[u'progress'])
-          if match:
-            image_progress = int(match.group(1))
+        if status_msg == 'Pushing':
+          if u'progressDetail' in status and u'id' in status:
+            image_id = status[u'id']
+            detail = status[u'progressDetail']
 
-        result_object['current_image'] = current_image
-        result_object['image_completion_percent'] = image_progress
+            if u'current' in detail and 'total' in detail:
+              images = result_object['image_completion']
+
+              images[image_id] = detail
+              result_object['push_completion'] = total_completion(images,
+                                                                  num_images)
 
       elif u'errorDetail' in status:
         result_object['status'] = 'error'
         if u'message' in status[u'errorDetail']:
-          result_object['message'] = status[u'errorDetail'][u'message']
+          result_object['message'] = str(status[u'errorDetail'][u'message'])
         return
 
     result_object['status'] = 'complete'
@@ -136,12 +139,11 @@ MIME_PROCESSORS = {
 
 build = {
   'total_commands': None,
-  'total_images': None,
   'current_command': None,
-  'current_image': None,
-  'image_completion_percent': None,
+  'push_completion': 0.0,
   'status': 'waiting',
   'message': None,
+  'image_completion': {},
 }
 pool = ThreadPool(1)
 

workers/dockerfilebuild.py

@@ -26,7 +26,8 @@ formatter = logging.Formatter(FORMAT)
 
 logger = logging.getLogger(__name__)
 
-BUILD_SERVER_CMD = ('docker run -d -lxc-conf="lxc.aa_profile=unconfined" ' +
+BUILD_SERVER_CMD = ('docker run -d -p 5002:5002 ' + 
+                    '-lxc-conf="lxc.aa_profile=unconfined" ' +
                     '-privileged -e \'RESOURCE_URL=%s\' -e \'TAG=%s\' ' +
                     '-e \'TOKEN=%s\' quay.io/quay/buildserver')
 
@@ -189,7 +190,7 @@ def babysit_builder(request):
     retry_command(droplet.destroy)
 
     repository_build.status_url = None
-    repository_build.build_node_id = None;
+    repository_build.build_node_id = None
     repository_build.save()
 
     return True