From c8cc7f1a19dcced4e0f7b21dd02d85a22f964803 Mon Sep 17 00:00:00 2001 From: kalomaze <66376113+kalomaze@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:46:30 -0500 Subject: [PATCH] Proposed streaming improvements --- koboldcpp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/koboldcpp.py b/koboldcpp.py index 403fe44d3..22e117bcb 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -428,7 +428,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): incomplete_token_buffer = bytearray() while not handle.has_finished(): - if current_token < handle.get_stream_count(): + while current_token < handle.get_stream_count(): token = handle.new_token(current_token) if token is None: # Token isnt ready yet, received nullpointer @@ -445,7 +445,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): event_str = json.dumps(event_data) await self.send_sse_event("message", event_str) - await asyncio.sleep(0) + await asyncio.sleep(0.1) # flush buffers, sleep a bit to make sure all data sent, and then force close the connection self.wfile.flush() @@ -1796,4 +1796,4 @@ if __name__ == '__main__': parser.add_argument("--gpulayers", help="Set number of layers to offload to GPU when using GPU. Requires GPU.",metavar=('[GPU layers]'), type=int, default=0) parser.add_argument("--tensor_split", help="For CUDA with ALL GPU set only, ratio to split tensors across multiple GPUs, space-separated list of proportions, e.g. 7 3", metavar=('[Ratios]'), type=float, nargs='+') - main(parser.parse_args(),start_server=True) \ No newline at end of file + main(parser.parse_args(),start_server=True)