Merge remote-tracking branch 'origin/master' into pullinprivate

Conflicts:
	workers/dockerfilebuild.py

workers/dockerfilebuild.py

@@ -49,9 +49,43 @@ class StatusWrapper(object):
     build_logs.set_status(self._uuid, self._status)
 
 
-def unwrap_stream(json_stream):
-  for json_entry in json_stream:
-    yield json.loads(json_entry).values()[0]
+class _IncompleteJsonError(Exception):
+  def __init__(self, start_from):
+    self.start_from = start_from
+
+
+class _StreamingJSONDecoder(json.JSONDecoder):
+  FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
+  WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
+
+  def decode(self, s, _w=WHITESPACE.match):
+    """Return the Python representation of ``s`` (a ``str`` or ``unicode``
+    instance containing a JSON document)
+
+    """
+    start_from = 0
+    while start_from < len(s):
+      try:
+        obj, end = self.raw_decode(s[start_from:], idx=_w(s[start_from:], 0).end())
+      except ValueError:
+        raise _IncompleteJsonError(start_from)
+      end = _w(s[start_from:], end).end()
+      start_from += end
+      yield obj
+
+
+class StreamingDockerClient(Client):
+  def _stream_helper(self, response):
+    """Generator for data coming from a chunked-encoded HTTP response."""
+    content_buf = ''
+    for content in response.iter_content(chunk_size=256):
+      content_buf += content
+      try:
+        for val in json.loads(content_buf, cls=_StreamingJSONDecoder):
+          yield val
+        content_buf = ''
+      except _IncompleteJsonError as exc:
+        content_buf = content_buf[exc.start_from:]
 
 
 class DockerfileBuildContext(object):
@@ -69,11 +103,11 @@ class DockerfileBuildContext(object):
     # Note: We have two different clients here because we (potentially) login
     # with both, but with different credentials that we do not want shared between
     # the build and push operations.
-    self._push_cl = Client(timeout=1200)
-    self._build_cl = Client(timeout=1200)
+    self._push_cl = StreamingDockerClient(timeout=1200)
+    self._build_cl = StreamingDockerClient(timeout=1200)
 
     dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
-                                   "Dockerfile")
+                                   'Dockerfile')
     self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
 
     logger.debug('Will build and push to repo %s with tags named: %s' %
@@ -108,7 +142,7 @@ class DockerfileBuildContext(object):
     # Login with the specified credentials (if any).
     if self._pull_credentials:
       logger.debug('Logging in with pull credentials.')
-      self.build_cl_.login(self._pull_credentials['username'], self._pull_credentials['password'],
+      self._build_cl.login(self._pull_credentials['username'], self._pull_credentials['password'],
                            registry=self._pull_credentials['registry'], reauth=True)
 
     # Start the build itself.
@@ -127,7 +161,7 @@ class DockerfileBuildContext(object):
 
     current_step = 0
     built_image = None
-    for status in unwrap_stream(build_status):
+    for status in build_status:
       fully_unwrapped = ""
       if isinstance(status, dict):
         if len(status) > 0:
@@ -189,9 +223,8 @@ class DockerfileBuildContext(object):
     logger.debug('Pushing to repo %s' % self._repo)
     resp = self._push_cl.push(self._repo, stream=True)
 
-    for status_str in resp:
-      status = json.loads(status_str)
-      logger.debug('Status: %s', status_str)
+    for status in resp:
+      logger.debug('Status: %s', status)
       if u'status' in status:
         status_msg = status[u'status']