vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

better error handling, try to avoid segfault in sillytavern

Browse source
This commit is contained in:
Concedo 2023-12-21 22:58:48 +08:00
parent c05d195583
commit 2378a29bde
1 changed files with 42 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

10
koboldcpp.py
View file

@ -568,6 +568,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
current_token = 0 current_token = 0
incomplete_token_buffer = bytearray() incomplete_token_buffer = bytearray()
await asyncio.sleep(0.05) #anti race condition, prevent check from overtaking generate await asyncio.sleep(0.05) #anti race condition, prevent check from overtaking generate
try:
while True: while True:
streamDone = handle.has_finished() #exit next loop on done streamDone = handle.has_finished() #exit next loop on done
tokenStr = "" tokenStr = ""
@ -605,6 +606,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
if api_format == 4: # if oai chat, send last [DONE] message consistent with openai format if api_format == 4: # if oai chat, send last [DONE] message consistent with openai format
await self.send_oai_sse_event('[DONE]') await self.send_oai_sse_event('[DONE]')
break break
except Exception as ex:
print("SSE streaming was interrupted due to an exception")
print(ex)
# flush buffers, sleep a bit to make sure all data sent, and then force close the connection # flush buffers, sleep a bit to make sure all data sent, and then force close the connection
self.wfile.flush() self.wfile.flush()
@ -629,7 +633,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
except (BrokenPipeError, ConnectionAbortedError) as cae: # attempt to abort if connection lost except (BrokenPipeError, ConnectionAbortedError) as cae: # attempt to abort if connection lost
print(cae) print(cae)
handle.abort_generate() handle.abort_generate()
time.sleep(0.1) #short delay time.sleep(0.2) #short delay
except Exception as e: except Exception as e:
print(e) print(e)
@ -938,8 +942,10 @@ Enter Prompt:<br>
self.send_header('content-length', str(len(genresp))) self.send_header('content-length', str(len(genresp)))
self.end_headers(content_type='application/json') self.end_headers(content_type='application/json')
self.wfile.write(genresp) self.wfile.write(genresp)
except: except Exception as ex:
print("Generate: The response could not be sent, maybe connection was terminated?") print("Generate: The response could not be sent, maybe connection was terminated?")
handle.abort_generate()
time.sleep(0.2) #short delay
return return
finally: finally:
modelbusy.release() modelbusy.release()