vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SimpleChat:WIP:Collate internally, Stream mode Trap exceptions

Browse source 
This can help ensure that data fetched till that point, can be
made use of, rather than losing it.

On some platforms, the time taken wrt generating a long response,
may lead to the network connection being broken when it enters
some user-no-interaction related power saving mode.
This commit is contained in:
HanishKVC 2024-05-30 03:51:28 +05:30
parent 009563d1d7
commit b75b3db7bf
1 changed files with 22 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

28
examples/server/public_simplechat/simplechat.js
View file

@ -71,6 +71,7 @@ class SimpleChat {
*/
this.xchat = [];
this.iLastSys = -1;
this.latestResponse = "";
}
clear() {
@ -122,6 +123,15 @@ class SimpleChat {
return rchat;
}
/**
* Collate the latest response from the server/ai-model, as it is becoming available.
* This is mainly useful for the stream mode.
* @param {string} content
*/
append_response(content) {
this.latestResponse += content;
}
/**
* Add an entry into xchat
* @param {string} role
@ -355,7 +365,7 @@ class SimpleChat {
}
let tdUtf8 = new TextDecoder("utf-8");
let rr = resp.body.getReader();
let gotBody = "";
this.latestResponse = "";
let xLines = new du.NewLines();
while(true) {
let { value: cur, done: done } = await rr.read();
@ -377,16 +387,16 @@ class SimpleChat {
}
let curJson = JSON.parse(curLine);
console.debug("DBUG:SC:PART:Json:", curJson);
gotBody += this.response_extract_stream(curJson, apiEP);
this.append_response(this.response_extract_stream(curJson, apiEP));
}
elP.innerText = gotBody;
elP.innerText = this.latestResponse;
elP.scrollIntoView(false);
if (done) {
break;
}
}
console.debug("DBUG:SC:PART:Full:", gotBody);
return gotBody;
console.debug("DBUG:SC:PART:Full:", this.latestResponse);
return this.latestResponse;
}
/**
@ -414,7 +424,13 @@ class SimpleChat {
}
let origMsg;
if (gMe.bStream) {
origMsg = await this.handle_response_multipart(resp, apiEP, elDiv);
try {
origMsg = await this.handle_response_multipart(resp, apiEP, elDiv);
this.latestResponse = "";
} catch (error) {
origMsg = this.latestResponse;
throw error;
}
} else {
origMsg = await this.handle_response_oneshot(resp, apiEP);
}