vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SimpleChat: show streamed generative text as it becomes available

Browse source 
Now that the extracting of streamed generated text is implemented,
add logic to show the same on the screen.
This commit is contained in:
HanishKVC 2024-05-28 10:15:15 +05:30
parent 08b117b4a7
commit 4d354556dc
2 changed files with 14 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

4
examples/server/public_simplechat/readme.md
View file

@ -11,6 +11,9 @@ in a simple way with minimal code from a common code base. Inturn additionally i
multiple independent back and forth chatting to an extent, with the ai llm model at a basic level, with their multiple independent back and forth chatting to an extent, with the ai llm model at a basic level, with their
own system prompts. own system prompts.
This allows seeing the generated text / ai-model response in oneshot at the end, after it is fully generated,
or potentially as it is being generated, in a streamed manner from the server/ai-model.
The UI follows a responsive web design so that the layout can adapt to available display space in a usable The UI follows a responsive web design so that the layout can adapt to available display space in a usable
enough manner, in general. enough manner, in general.
@ -58,6 +61,7 @@ Once inside
* chat (default) vs completion mode * chat (default) vs completion mode
* try trim garbage in response or not * try trim garbage in response or not
* amount of chat history in the context sent to server/ai-model * amount of chat history in the context sent to server/ai-model
* oneshot or streamed mode.
* In completion mode * In completion mode
* logic by default doesnt insert any role specific "ROLE: " prefix wrt each role's message. * logic by default doesnt insert any role specific "ROLE: " prefix wrt each role's message.

14
examples/server/public_simplechat/simplechat.js
View file

@ -150,6 +150,7 @@ class SimpleChat {
gMe.show_info(div); gMe.show_info(div);
} }
} }
return last;
} }
/** /**
@ -316,8 +317,10 @@ class SimpleChat {
* Handle the multipart response from server/ai-model * Handle the multipart response from server/ai-model
* @param {Response} resp * @param {Response} resp
* @param {string} apiEP * @param {string} apiEP
* @param {HTMLDivElement} elDiv
*/ */
async handle_response_multipart(resp, apiEP) { async handle_response_multipart(resp, apiEP, elDiv) {
let elP = ui.el_create_append_p("", elDiv);
if (!resp.body) { if (!resp.body) {
throw Error("ERRR:SimpleChat:SC:ReadJsonEarly:No body..."); throw Error("ERRR:SimpleChat:SC:ReadJsonEarly:No body...");
} }
@ -343,6 +346,8 @@ class SimpleChat {
gotBody += this.response_extract_stream(curJson, apiEP); gotBody += this.response_extract_stream(curJson, apiEP);
} }
} }
elP.innerText = gotBody;
elP.scrollIntoView(false);
if (done) { if (done) {
break; break;
} }
@ -367,15 +372,16 @@ class SimpleChat {
* Also take care of the optional garbage trimming. * Also take care of the optional garbage trimming.
* @param {Response} resp * @param {Response} resp
* @param {string} apiEP * @param {string} apiEP
* @param {HTMLDivElement} elDiv
*/ */
async handle_response(resp, apiEP) { async handle_response(resp, apiEP, elDiv) {
let theResp = { let theResp = {
assistant: "", assistant: "",
trimmed: "", trimmed: "",
} }
let origMsg; let origMsg;
if (gMe.bStream) { if (gMe.bStream) {
origMsg = await this.handle_response_multipart(resp, apiEP); origMsg = await this.handle_response_multipart(resp, apiEP, elDiv);
} else { } else {
origMsg = await this.handle_response_oneshot(resp, apiEP); origMsg = await this.handle_response_oneshot(resp, apiEP);
} }
@ -546,7 +552,7 @@ class MultiChatUI {
body: theBody, body: theBody,
}); });
let theResp = await chat.handle_response(resp, apiEP); let theResp = await chat.handle_response(resp, apiEP, this.elDivChat);
chat.add(Roles.Assistant, theResp.assistant); chat.add(Roles.Assistant, theResp.assistant);
if (chatId == this.curChatId) { if (chatId == this.curChatId) {
chat.show(this.elDivChat); chat.show(this.elDivChat);