vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tidy up the new sse stream code

Browse source
This commit is contained in:
Concedo 2023-09-07 10:48:53 +08:00
parent 49089371d1
commit a8673be66e
1 changed files with 5 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
koboldcpp.py
View file

@ -434,11 +434,12 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
incomplete_token_buffer = bytearray() incomplete_token_buffer = bytearray()
while not handle.has_finished(): while not handle.has_finished():
while current_token < handle.get_stream_count(): streamcount = handle.get_stream_count()
while current_token < streamcount:
token = handle.new_token(current_token) token = handle.new_token(current_token)
if token is None: # Token isnt ready yet, received nullpointer if token is None: # Token isnt ready yet, received nullpointer
continue break
current_token += 1 current_token += 1
@ -451,7 +452,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
event_str = json.dumps(event_data) event_str = json.dumps(event_data)
await self.send_sse_event("message", event_str) await self.send_sse_event("message", event_str)
await asyncio.sleep(0.1) await asyncio.sleep(0.02) #this should keep things responsive
# flush buffers, sleep a bit to make sure all data sent, and then force close the connection # flush buffers, sleep a bit to make sure all data sent, and then force close the connection
self.wfile.flush() self.wfile.flush()
@ -1803,4 +1804,4 @@ if __name__ == '__main__':
parser.add_argument("--gpulayers", help="Set number of layers to offload to GPU when using GPU. Requires GPU.",metavar=('[GPU layers]'), type=int, default=0) parser.add_argument("--gpulayers", help="Set number of layers to offload to GPU when using GPU. Requires GPU.",metavar=('[GPU layers]'), type=int, default=0)
parser.add_argument("--tensor_split", help="For CUDA with ALL GPU set only, ratio to split tensors across multiple GPUs, space-separated list of proportions, e.g. 7 3", metavar=('[Ratios]'), type=float, nargs='+') parser.add_argument("--tensor_split", help="For CUDA with ALL GPU set only, ratio to split tensors across multiple GPUs, space-separated list of proportions, e.g. 7 3", metavar=('[Ratios]'), type=float, nargs='+')
main(parser.parse_args(),start_server=True) main(parser.parse_args(),start_server=True)