vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SimpleChat:Stream:Initial handshake skeleton

Browse source 
Parse the got stream responses and try extract the data from it.

It allows for a part read to get a single data line or multiple
data line. Inturn extract the json body and inturn the delta
content/message in it.
This commit is contained in:
HanishKVC 2024-05-28 06:06:16 +05:30
parent 060925cda3
commit 9d0e65d16a
1 changed files with 44 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

49
examples/server/public_simplechat/simplechat.js
View file

@ -234,6 +234,27 @@ class SimpleChat {
return theResp;
}
/**
* Extract the ai-model/assistant's response from the http response got in streaming mode.
* @param {any} respBody
* @param {string} apiEP
*/
response_extract_stream(respBody, apiEP) {
let assistant = "";
if (apiEP == ApiEP.Type.Chat) {
if (respBody["choices"][0]["finish_reason"] !== "stop") {
assistant = respBody["choices"][0]["delta"]["content"];
}
} else {
try {
assistant = respBody["choices"][0]["text"];
} catch {
assistant = respBody["content"];
}
}
return assistant;
}
/**
* Allow setting of system prompt, but only at begining.
* @param {string} sysPrompt
@ -411,9 +432,10 @@ class MultiChatUI {
/**
* Try read json response early, if available.
* @param {SimpleChat} chat
* @param {Response} resp
*/
async read_json_early(resp) {
async read_json_early(chat, resp) {
if (!resp.body) {
throw Error("ERRR:SimpleChat:MCUI:ReadJsonEarly:No body...");
}
@ -423,13 +445,28 @@ class MultiChatUI {
while(true) {
let { value: cur, done: done } = await rr.read();
let curBody = tdUtf8.decode(cur);
console.debug("DBUG:SC:PART:", curBody);
gotBody += curBody;
console.debug("DBUG:SC:PART:Str:", curBody);
if (curBody.length > 0) {
let curArrays = curBody.split("\n");
for(let curArray of curArrays) {
console.debug("DBUG:SC:PART:StrPart:", curArray);
if (curArray.length <= 0) {
continue;
}
if (curArray.startsWith("data:")) {
curArray = curArray.substring(5);
}
let curJson = JSON.parse(curArray);
console.debug("DBUG:SC:PART:Json:", curJson);
gotBody += chat.response_extract_stream(curJson, gMe.apiEP);
}
}
if (done) {
break;
}
}
return JSON.parse(gotBody);
console.debug("DBUG:SC:PART:Full:", gotBody);
return gotBody;
}
/**
@ -472,8 +509,8 @@ class MultiChatUI {
body: theBody,
});
let respBody = await resp.json();
//let respBody = await this.read_json_early(resp);
//let respBody = await resp.json();
let respBody = await this.read_json_early(chat, resp);
console.debug(`DBUG:SimpleChat:MCUI:${chatId}:HandleUserSubmit:RespBody:${JSON.stringify(respBody)}`);
let theResp = chat.response_extract(respBody, apiEP);
chat.add(Roles.Assistant, theResp.assistant);