vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

only start a new thread when using sse

Browse source
This commit is contained in:
Concedo 2023-06-10 19:03:41 +08:00
parent 43f7e40470
commit a68fcfe738
1 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

11
koboldcpp.py
View file

@ -234,8 +234,6 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
async def generate_text(self, newprompt, genparams, basic_api_flag, stream_flag): async def generate_text(self, newprompt, genparams, basic_api_flag, stream_flag):
loop = asyncio.get_event_loop()
executor = ThreadPoolExecutor()
def run_blocking(): def run_blocking():
if basic_api_flag: if basic_api_flag:
@ -270,8 +268,13 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
stop_sequence=genparams.get('stop_sequence', []), stop_sequence=genparams.get('stop_sequence', []),
stream_sse=stream_flag) stream_sse=stream_flag)
recvtxt = ""
recvtxt = await loop.run_in_executor(executor, run_blocking) if stream_flag:
loop = asyncio.get_event_loop()
executor = ThreadPoolExecutor()
recvtxt = await loop.run_in_executor(executor, run_blocking)
else:
recvtxt = run_blocking()
utfprint("\nOutput: " + recvtxt) utfprint("\nOutput: " + recvtxt)