vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SimpleChat:Stream:Initial handshake skeleton

Browse source 
Parse the got stream responses and try extract the data from it.

It allows for a part read to get a single data line or multiple
data line. Inturn extract the json body and inturn the delta
content/message in it.
This commit is contained in:
HanishKVC 2024-05-28 06:06:16 +05:30
parent 060925cda3
commit 9d0e65d16a
1 changed files with 44 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

51
examples/server/public_simplechat/simplechat.js
View file

@ -234,6 +234,27 @@ class SimpleChat {
return theResp; return theResp;
} }
/**
* Extract the ai-model/assistant's response from the http response got in streaming mode.
* @param {any} respBody
* @param {string} apiEP
*/
response_extract_stream(respBody, apiEP) {
let assistant = "";
if (apiEP == ApiEP.Type.Chat) {
if (respBody["choices"][0]["finish_reason"] !== "stop") {
assistant = respBody["choices"][0]["delta"]["content"];
}
} else {
try {
assistant = respBody["choices"][0]["text"];
} catch {
assistant = respBody["content"];
}
}
return assistant;
}
/** /**
* Allow setting of system prompt, but only at begining. * Allow setting of system prompt, but only at begining.
* @param {string} sysPrompt * @param {string} sysPrompt
@ -411,9 +432,10 @@ class MultiChatUI {
/** /**
* Try read json response early, if available. * Try read json response early, if available.
* @param {SimpleChat} chat
* @param {Response} resp * @param {Response} resp
*/ */
async read_json_early(resp) { async read_json_early(chat, resp) {
if (!resp.body) { if (!resp.body) {
throw Error("ERRR:SimpleChat:MCUI:ReadJsonEarly:No body..."); throw Error("ERRR:SimpleChat:MCUI:ReadJsonEarly:No body...");
} }
@ -421,15 +443,30 @@ class MultiChatUI {
let rr = resp.body.getReader(); let rr = resp.body.getReader();
let gotBody = ""; let gotBody = "";
while(true) { while(true) {
let { value: cur, done: done} = await rr.read(); let { value: cur, done: done } = await rr.read();
let curBody = tdUtf8.decode(cur); let curBody = tdUtf8.decode(cur);
console.debug("DBUG:SC:PART:", curBody); console.debug("DBUG:SC:PART:Str:", curBody);
gotBody += curBody; if (curBody.length > 0) {
let curArrays = curBody.split("\n");
for(let curArray of curArrays) {
console.debug("DBUG:SC:PART:StrPart:", curArray);
if (curArray.length <= 0) {
continue;
}
if (curArray.startsWith("data:")) {
curArray = curArray.substring(5);
}
let curJson = JSON.parse(curArray);
console.debug("DBUG:SC:PART:Json:", curJson);
gotBody += chat.response_extract_stream(curJson, gMe.apiEP);
}
}
if (done) { if (done) {
break; break;
} }
} }
return JSON.parse(gotBody); console.debug("DBUG:SC:PART:Full:", gotBody);
return gotBody;
} }
/** /**
@ -472,8 +509,8 @@ class MultiChatUI {
body: theBody, body: theBody,
}); });
let respBody = await resp.json(); //let respBody = await resp.json();
//let respBody = await this.read_json_early(resp); let respBody = await this.read_json_early(chat, resp);
console.debug(`DBUG:SimpleChat:MCUI:${chatId}:HandleUserSubmit:RespBody:${JSON.stringify(respBody)}`); console.debug(`DBUG:SimpleChat:MCUI:${chatId}:HandleUserSubmit:RespBody:${JSON.stringify(respBody)}`);
let theResp = chat.response_extract(respBody, apiEP); let theResp = chat.response_extract(respBody, apiEP);
chat.add(Roles.Assistant, theResp.assistant); chat.add(Roles.Assistant, theResp.assistant);