vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Proposed streaming improvements

Browse source
This commit is contained in:
kalomaze 2023-09-05 16:46:30 -05:00 • committed by GitHub
parent b6914ebd04
commit c8cc7f1a19
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
koboldcpp.py
View file

@ -428,7 +428,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
incomplete_token_buffer = bytearray()
while not handle.has_finished():
if current_token < handle.get_stream_count():
while current_token < handle.get_stream_count():
token = handle.new_token(current_token)
if token is None: # Token isnt ready yet, received nullpointer
@ -445,7 +445,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
event_str = json.dumps(event_data)
await self.send_sse_event("message", event_str)
await asyncio.sleep(0)
await asyncio.sleep(0.1)
# flush buffers, sleep a bit to make sure all data sent, and then force close the connection
self.wfile.flush()