From 6a821b268a78d712d695834a34eb43dc480aa4e6 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:33:34 +0800 Subject: [PATCH] improved SSE streamiing --- klite.embd | 51 ++++++++++++++++++++++++++++++++------------------- koboldcpp.py | 24 ++++++++++++++++-------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/klite.embd b/klite.embd index 19c2b2a9c..c6c743b6e 100644 --- a/klite.embd +++ b/klite.embd @@ -2959,7 +2959,7 @@ Current version: 71 instruct_starttag: "\\n### Instruction:\\n", instruct_endtag: "\\n### Response:\\n", instruct_has_markdown: false, - raw_instruct_tags: false, + placeholder_tags: true, persist_session: true, speech_synth: 0, //0 is disabled beep_on: false, @@ -5144,8 +5144,11 @@ Current version: 71 { desired_oai_ep = desired_oai_ep.slice(0, -1); } - if(desired_oai_ep!="" && desired_oai_ep.length > 4 && !desired_oai_ep.slice(-4).toLowerCase().includes("/v")) { - desired_oai_ep = desired_oai_ep + "/v1"; + if (document.getElementById("oaiaddversion").checked) + { + if(desired_oai_ep!="" && desired_oai_ep.length > 4 && !desired_oai_ep.slice(-4).toLowerCase().includes("/v")) { + desired_oai_ep = desired_oai_ep + "/v1"; + } } if(desired_oai_key!="" && desired_oai_ep!="") { @@ -5288,8 +5291,11 @@ Current version: 71 { desired_claude_ep = desired_claude_ep.slice(0, -1); } - if(desired_claude_ep!="" && desired_claude_ep.length > 4 && !desired_claude_ep.slice(-4).toLowerCase().includes("/v")) { - desired_claude_ep = desired_claude_ep + "/v1"; + if (document.getElementById("claudeaddversion").checked) + { + if (desired_claude_ep != "" && desired_claude_ep.length > 4 && !desired_claude_ep.slice(-4).toLowerCase().includes("/v")) { + desired_claude_ep = desired_claude_ep + "/v1"; + } } if(desired_claude_key!="" && desired_claude_ep!="") { @@ -5815,7 +5821,7 @@ Current version: 71 document.getElementById("idle_duration").value = localsettings.idle_duration; document.getElementById("adventure_context_mod").checked = localsettings.adventure_context_mod; document.getElementById("instruct_has_markdown").checked = localsettings.instruct_has_markdown; - document.getElementById("raw_instruct_tags").checked = localsettings.raw_instruct_tags; + document.getElementById("placeholder_tags").checked = localsettings.placeholder_tags; document.getElementById("auto_ctxlen").checked = localsettings.auto_ctxlen; document.getElementById("auto_genamt").checked = localsettings.auto_genamt; if(localflag) @@ -5994,7 +6000,7 @@ Current version: 71 localsettings.idle_duration = document.getElementById("idle_duration").value; localsettings.adventure_context_mod = (document.getElementById("adventure_context_mod").checked ? true : false); localsettings.instruct_has_markdown = (document.getElementById("instruct_has_markdown").checked ? true : false); - localsettings.raw_instruct_tags = (document.getElementById("raw_instruct_tags").checked ? true : false); + localsettings.placeholder_tags = (document.getElementById("placeholder_tags").checked ? true : false); localsettings.generate_images = document.getElementById("generate_images").value; localsettings.opmode = document.getElementById("opmode").value; localsettings.chatname = document.getElementById("chatname").value; @@ -6382,7 +6388,7 @@ Current version: 71 function replace_placeholders(inputtxt) { //only do this for chat and instruct modes - if(localsettings.opmode==3||localsettings.opmode==4) + if(localsettings.placeholder_tags) { inputtxt = replaceAll(inputtxt,"{{user}}",localsettings.chatname?localsettings.chatname:"You",true); inputtxt = replaceAll(inputtxt,"{{char}}",localsettings.chatopponent?localsettings.chatopponent:defaultchatopponent,true); @@ -6583,7 +6589,7 @@ Current version: 71 if(newgen != "") { //append instruction for instruct mode - if (localsettings.raw_instruct_tags) { + if (!localsettings.placeholder_tags) { newgen = get_instruct_starttag(false) + newgen + get_instruct_endtag(false); } else { @@ -6593,7 +6599,7 @@ Current version: 71 else //may be continuting existing instruction OR starting a brand new session. check if first action { if (gametext_arr.length == 0) { - if (localsettings.raw_instruct_tags) { + if (!localsettings.placeholder_tags) { newgen = get_instruct_endtag(false); //bot response as first msg } else { newgen = instructendplaceholder; @@ -7560,12 +7566,6 @@ Current version: 71 //handle stopping tokens if they got missed (eg. horde) gentxt = trim_extra_stop_seqs(gentxt,true); - //trim trailing whitespace, and multiple newlines - if (localsettings.trimwhitespace) { - gentxt = gentxt.replace(/[\t\r\n ]+$/, ''); - gentxt = gentxt.replace(/[\r\n]+/g, '\n'); - } - //always trim incomplete sentences for adventure and chat (if not multiline) if (localsettings.opmode == 2 || (localsettings.opmode == 3 && !localsettings.allow_continue_chat) || localsettings.trimsentences == true) { gentxt = end_trim_to_sentence(gentxt,true); @@ -7574,6 +7574,12 @@ Current version: 71 //do a second pass, this time removing the actual stop token gentxt = trim_extra_stop_seqs(gentxt,false); + //trim trailing whitespace, and multiple newlines + if (localsettings.trimwhitespace) { + gentxt = gentxt.replace(/[\t\r\n ]+$/, ''); + gentxt = gentxt.replace(/[\r\n]+/g, '\n'); + } + //if we are in adventure mode, truncate to action if it appears if (localsettings.opmode == 2) { @@ -9753,6 +9759,8 @@ Current version: 71 + +
Add Endpoint Version
Improve Prompt (System Message Injection)
claude-2 + +
Add Endpoint Version
Claude Compatibility Rename Fix
@@ -10032,9 +10042,7 @@ Current version: 71 -
Raw Instruct Tags ?If enabled, uses instruct tags verbatim. If disabled, uses universal instruct placeholders that get swapped on submit.
- +
Enable Markdown ?Allows the UI to use markdown formatting such as quotes and code blocks.
@@ -10213,6 +10221,11 @@ Current version: 71
Unban EOS Tokens
+
+
Placeholder Tags ?If enabled, uses universal {{user}} and {{[INPUT]}} placeholders that get swapped on submit. If disabled, uses plaintext chat or instruct tags verbatim.
+ +
diff --git a/koboldcpp.py b/koboldcpp.py index 23008e911..50dcd9ea0 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -461,7 +461,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): current_token = 0 incomplete_token_buffer = bytearray() - while not handle.has_finished(): + while True: + streamDone = handle.has_finished() #exit next loop on done + tokenStr = "" streamcount = handle.get_stream_count() while current_token < streamcount: token = handle.new_token(current_token) @@ -470,17 +472,23 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler): break current_token += 1 - newbyte = ctypes.string_at(token) incomplete_token_buffer += bytearray(newbyte) - tokenStr = incomplete_token_buffer.decode("UTF-8","ignore") - if tokenStr!="": + tokenSeg = incomplete_token_buffer.decode("UTF-8","ignore") + if tokenSeg!="": incomplete_token_buffer.clear() - event_data = {"token": tokenStr} - event_str = json.dumps(event_data) - await self.send_sse_event("message", event_str) + tokenStr += tokenSeg - await asyncio.sleep(0.02) #this should keep things responsive + if tokenStr!="": + event_data = {"token": tokenStr} + event_str = json.dumps(event_data) + tokenStr = "" + await self.send_sse_event("message", event_str) + else: + await asyncio.sleep(0.02) #this should keep things responsive + + if streamDone: + break # flush buffers, sleep a bit to make sure all data sent, and then force close the connection self.wfile.flush()