Freeze the version of docker-py to 0.3.0 and create a fancy streaming json decoder to work around the lack of newlines in push statuses.

requirements-nover.txt

@@ -21,7 +21,7 @@ xhtml2pdf
 logstash_formatter
 redis
 hiredis
-git+https://github.com/dotcloud/docker-py.git
+docker-py
 loremipsum
 pygithub
 flask-restful

requirements.txt

@@ -16,7 +16,7 @@ beautifulsoup4==4.3.2
 blinker==1.3
 boto==2.27.0
 distribute==0.6.34
-git+https://github.com/dotcloud/docker-py.git
+docker-py==0.3.0
 ecdsa==0.11
 gevent==1.0
 greenlet==0.4.2

workers/dockerfilebuild.py

@@ -49,12 +49,43 @@ class StatusWrapper(object):
     build_logs.set_status(self._uuid, self._status)
 
 
-def unwrap_stream(json_stream):
+class _IncompleteJsonError(Exception):
-  for json_entry in json_stream:
+  def __init__(self, start_from):
-    try:
+    self.start_from = start_from
-      yield json.loads(json_entry).values()[0]
+
-    except ValueError:
+
-      logger.debug('Invalid json block: %s' % json_entry)
+class _StreamingJSONDecoder(json.JSONDecoder):
+  FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
+  WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
+
+  def decode(self, s, _w=WHITESPACE.match):
+    """Return the Python representation of ``s`` (a ``str`` or ``unicode``
+    instance containing a JSON document)
+
+    """
+    start_from = 0
+    while start_from < len(s):
+      try:
+        obj, end = self.raw_decode(s[start_from:], idx=_w(s[start_from:], 0).end())
+      except ValueError:
+        raise _IncompleteJsonError(start_from)
+      end = _w(s[start_from:], end).end()
+      start_from += end
+      yield obj
+
+
+class StreamingDockerClient(Client):
+  def _stream_helper(self, response):
+    """Generator for data coming from a chunked-encoded HTTP response."""
+    content_buf = ''
+    for content in response.iter_content(chunk_size=256):
+      content_buf += content
+      try:
+        for val in json.loads(content_buf, cls=_StreamingJSONDecoder):
+          yield val
+        content_buf = ''
+      except _IncompleteJsonError as exc:
+        content_buf = content_buf[exc.start_from:]
 
 
 class DockerfileBuildContext(object):
@@ -65,12 +96,12 @@ class DockerfileBuildContext(object):
     self._repo = repo
     self._tag_names = tag_names
     self._push_token = push_token
-    self._cl = Client(timeout=1200)
+    self._cl = StreamingDockerClient(timeout=1200)
     self._status = StatusWrapper(build_uuid)
     self._build_logger = partial(build_logs.append_log_message, build_uuid)
 
     dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
-                                   "Dockerfile")
+                                   'Dockerfile')
     self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
 
     logger.debug('Will build and push to repo %s with tags named: %s' %
@@ -117,7 +148,7 @@ class DockerfileBuildContext(object):
 
     current_step = 0
     built_image = None
-    for status in unwrap_stream(build_status):
+    for status in build_status:
       fully_unwrapped = ""
       if isinstance(status, dict):
         if len(status) > 0:
@@ -179,9 +210,8 @@ class DockerfileBuildContext(object):
     logger.debug('Pushing to repo %s' % self._repo)
     resp = self._cl.push(self._repo, stream=True)
 
-    for status_str in resp:
+    for status in resp:
-      status = json.loads(status_str)
+      logger.debug('Status: %s', status)
-      logger.debug('Status: %s', status_str)
       if u'status' in status:
         status_msg = status[u'status']