vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merging Proper SSE Token Streaming #220 with end connection fix test

Browse source
This commit is contained in:
Concedo 2023-06-09 23:22:16 +08:00
parent b92f9fe3a2
commit 5bd9cef9fa
1 changed files with 6 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

8
koboldcpp.py
View file

@ -289,7 +289,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.send_header("Connection", "keep-alive") self.send_header("Connection", "keep-alive")
self.end_headers() self.end_headers()
current_token = 0; current_token = 0
while not handle.has_finished(): while not handle.has_finished():
if current_token < handle.get_stream_count(): if current_token < handle.get_stream_count():
@ -307,7 +307,10 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
await asyncio.sleep(0) await asyncio.sleep(0)
# Implement connection closing here # flush buffers, sleep a bit to make sure all data sent, and then force close the connection
self.wfile.flush()
await asyncio.sleep(0.1)
self.close_connection = True
async def handle_request(self, genparams, newprompt, basic_api_flag, stream_flag): async def handle_request(self, genparams, newprompt, basic_api_flag, stream_flag):
@ -417,6 +420,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
try: try:
genparams = json.loads(body) genparams = json.loads(body)
except ValueError as e: except ValueError as e:
utfprint("Body Err: " + str(body))
return self.send_response(503) return self.send_response(503)
utfprint("\nInput: " + json.dumps(genparams)) utfprint("\nInput: " + json.dumps(genparams))