diff --git a/requirements-nover.txt b/requirements-nover.txt index 39142b7c5..bac4ba690 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -21,7 +21,7 @@ xhtml2pdf logstash_formatter redis hiredis -git+https://github.com/dotcloud/docker-py.git +docker-py loremipsum pygithub flask-restful diff --git a/requirements.txt b/requirements.txt index 2d26ab24b..0b942eee1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ beautifulsoup4==4.3.2 blinker==1.3 boto==2.27.0 distribute==0.6.34 -git+https://github.com/dotcloud/docker-py.git +docker-py==0.3.0 ecdsa==0.11 gevent==1.0 greenlet==0.4.2 diff --git a/workers/dockerfilebuild.py b/workers/dockerfilebuild.py index 11cfcd773..7e75fc429 100644 --- a/workers/dockerfilebuild.py +++ b/workers/dockerfilebuild.py @@ -49,12 +49,43 @@ class StatusWrapper(object): build_logs.set_status(self._uuid, self._status) -def unwrap_stream(json_stream): - for json_entry in json_stream: - try: - yield json.loads(json_entry).values()[0] - except ValueError: - logger.debug('Invalid json block: %s' % json_entry) +class _IncompleteJsonError(Exception): + def __init__(self, start_from): + self.start_from = start_from + + +class _StreamingJSONDecoder(json.JSONDecoder): + FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) + + def decode(self, s, _w=WHITESPACE.match): + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + + """ + start_from = 0 + while start_from < len(s): + try: + obj, end = self.raw_decode(s[start_from:], idx=_w(s[start_from:], 0).end()) + except ValueError: + raise _IncompleteJsonError(start_from) + end = _w(s[start_from:], end).end() + start_from += end + yield obj + + +class StreamingDockerClient(Client): + def _stream_helper(self, response): + """Generator for data coming from a chunked-encoded HTTP response.""" + content_buf = '' + for content in response.iter_content(chunk_size=256): + content_buf += content + try: + for val in json.loads(content_buf, cls=_StreamingJSONDecoder): + yield val + content_buf = '' + except _IncompleteJsonError as exc: + content_buf = content_buf[exc.start_from:] class DockerfileBuildContext(object): @@ -65,12 +96,12 @@ class DockerfileBuildContext(object): self._repo = repo self._tag_names = tag_names self._push_token = push_token - self._cl = Client(timeout=1200) + self._cl = StreamingDockerClient(timeout=1200) self._status = StatusWrapper(build_uuid) self._build_logger = partial(build_logs.append_log_message, build_uuid) dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir, - "Dockerfile") + 'Dockerfile') self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path) logger.debug('Will build and push to repo %s with tags named: %s' % @@ -117,7 +148,7 @@ class DockerfileBuildContext(object): current_step = 0 built_image = None - for status in unwrap_stream(build_status): + for status in build_status: fully_unwrapped = "" if isinstance(status, dict): if len(status) > 0: @@ -179,9 +210,8 @@ class DockerfileBuildContext(object): logger.debug('Pushing to repo %s' % self._repo) resp = self._cl.push(self._repo, stream=True) - for status_str in resp: - status = json.loads(status_str) - logger.debug('Status: %s', status_str) + for status in resp: + logger.debug('Status: %s', status) if u'status' in status: status_msg = status[u'status']