vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'ycros/api-modelbusy-fix' into concedo_experimental

Browse source
This commit is contained in:
Concedo 2023-07-19 18:32:13 +08:00
commit 2a88d6d3ec
1 changed files with 34 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

13
koboldcpp.py
View file

@ -280,7 +280,7 @@ friendlymodelname = "concedo/koboldcpp" # local kobold api apparently needs a h
maxctx = 2048 maxctx = 2048
maxhordectx = 1024 maxhordectx = 1024
maxhordelen = 256 maxhordelen = 256
modelbusy = False modelbusy = threading.Lock()
defaultport = 5001 defaultport = 5001
KcppVersion = "1.36" KcppVersion = "1.36"
showdebug = True showdebug = True
@ -373,7 +373,6 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
async def handle_sse_stream(self): async def handle_sse_stream(self):
self.send_response(200) self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache") self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "keep-alive") self.send_header("Connection", "keep-alive")
self.end_headers() self.end_headers()
@ -495,7 +494,6 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.end_headers() self.end_headers()
self.wfile.write(json.dumps({"success": ("true" if ag else "false")}).encode()) self.wfile.write(json.dumps({"success": ("true" if ag else "false")}).encode())
print("\nGeneration Aborted") print("\nGeneration Aborted")
modelbusy = False
return return
if self.path.endswith('/api/extra/generate/check'): if self.path.endswith('/api/extra/generate/check'):
@ -506,7 +504,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.wfile.write(json.dumps({"results": [{"text": pendtxtStr}]}).encode()) self.wfile.write(json.dumps({"results": [{"text": pendtxtStr}]}).encode())
return return
if modelbusy: if not modelbusy.acquire(blocking=False):
self.send_response(503) self.send_response(503)
self.end_headers() self.end_headers()
self.wfile.write(json.dumps({"detail": { self.wfile.write(json.dumps({"detail": {
@ -515,6 +513,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
}}).encode()) }}).encode())
return return
try:
if self.path.endswith('/request'): if self.path.endswith('/request'):
basic_api_flag = True basic_api_flag = True
@ -536,8 +535,6 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
if args.debugmode!=-1: if args.debugmode!=-1:
utfprint("\nInput: " + json.dumps(genparams)) utfprint("\nInput: " + json.dumps(genparams))
modelbusy = True
if kai_api_flag: if kai_api_flag:
fullprompt = genparams.get('prompt', "") fullprompt = genparams.get('prompt', "")
else: else:
@ -552,9 +549,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
except: except:
print("Generate: The response could not be sent, maybe connection was terminated?") print("Generate: The response could not be sent, maybe connection was terminated?")
modelbusy = False
return return
finally:
modelbusy.release()
self.send_response(404) self.send_response(404)
self.end_headers() self.end_headers()