From 08b117b4a7289b745860fefab508f924d830b583 Mon Sep 17 00:00:00 2001
From: HanishKVC <hanishkvc@gmail.com>
Date: Tue, 28 May 2024 09:20:35 +0530
Subject: [PATCH] SimpleChat: Add MultiPart Response handling, common trimming

Add logic to call into multipart/stream server response handling.

Move trimming of garbage at the end into the common handle_response
helper.

Add new global flag to control between oneshot and multipart/stream
mode of fetching response. Allow same to be controlled by user.

If in multipart/stream mode, send the stream flag to the server.
---
 .../server/public_simplechat/simplechat.js    | 59 +++++++++++++------
 1 file changed, 40 insertions(+), 19 deletions(-)

diff --git a/examples/server/public_simplechat/simplechat.js b/examples/server/public_simplechat/simplechat.js
index d1d9ef47a..b416f7383 100644
--- a/examples/server/public_simplechat/simplechat.js
+++ b/examples/server/public_simplechat/simplechat.js
@@ -155,6 +155,7 @@ class SimpleChat {
     /**
      * Add needed fields wrt json object to be sent wrt LLM web services completions endpoint.
      * The needed fields/options are picked from a global object.
+     * Add optional stream flag, if required.
      * Convert the json into string.
      * @param {Object} obj
      */
@@ -162,6 +163,9 @@ class SimpleChat {
         for(let k in gMe.chatRequestOptions) {
             obj[k] = gMe.chatRequestOptions[k];
         }
+        if (gMe.bStream) {
+            obj["stream"] = true;
+        }
         return JSON.stringify(obj);
     }
 
@@ -217,25 +221,17 @@ class SimpleChat {
      * @param {string} apiEP
      */
     response_extract(respBody, apiEP) {
-        let theResp = {
-            assistant: "",
-            trimmed: "",
-        }
+        let assistant = "";
         if (apiEP == ApiEP.Type.Chat) {
-            theResp.assistant = respBody["choices"][0]["message"]["content"];
+            assistant = respBody["choices"][0]["message"]["content"];
         } else {
             try {
-                theResp.assistant = respBody["choices"][0]["text"];
+                assistant = respBody["choices"][0]["text"];
             } catch {
-                theResp.assistant = respBody["content"];
+                assistant = respBody["content"];
             }
         }
-        if (gMe.bTrimGarbage) {
-            let origMsg = theResp.assistant;
-            theResp.assistant = du.trim_garbage_at_end(theResp.assistant);
-            theResp.trimmed = origMsg.substring(theResp.assistant.length);
-        }
-        return theResp;
+        return assistant;
     }
 
     /**
@@ -344,7 +340,7 @@ class SimpleChat {
                     }
                     let curJson = JSON.parse(curArray);
                     console.debug("DBUG:SC:PART:Json:", curJson);
-                    gotBody += chat.response_extract_stream(curJson, gMe.apiEP);
+                    gotBody += this.response_extract_stream(curJson, apiEP);
                 }
             }
             if (done) {
@@ -366,10 +362,28 @@ class SimpleChat {
         return this.response_extract(respBody, apiEP);
     }
 
+    /**
+     * Handle the response from the server be it in oneshot or multipart/stream mode.
+     * Also take care of the optional garbage trimming.
+     * @param {Response} resp
+     * @param {string} apiEP
+     */
     async handle_response(resp, apiEP) {
-        if (apiEP == ApiEP.Type.Chat) {
-            return this.handle_response_oneshot(resp, apiEP);
+        let theResp = {
+            assistant: "",
+            trimmed: "",
         }
+        let origMsg;
+        if (gMe.bStream) {
+            origMsg = await this.handle_response_multipart(resp, apiEP);
+        } else {
+            origMsg = await this.handle_response_oneshot(resp, apiEP);
+        }
+        if (gMe.bTrimGarbage) {
+            theResp.assistant = du.trim_garbage_at_end(origMsg);
+            theResp.trimmed = origMsg.substring(theResp.assistant.length);
+        }
+        return theResp;
     }
 
 }
@@ -532,8 +546,7 @@ class MultiChatUI {
             body: theBody,
         });
 
-        let respBody = await this.read_json_early(chat, resp);
-        let theResp = chat.handle_response(respBody, apiEP);
+        let theResp = await chat.handle_response(resp, apiEP);
         chat.add(Roles.Assistant, theResp.assistant);
         if (chatId == this.curChatId) {
             chat.show(this.elDivChat);
@@ -628,6 +641,7 @@ class Me {
     constructor() {
         this.defaultChatIds = [ "Default", "Other" ];
         this.multiChat = new MultiChatUI();
+        this.bStream = true;
         this.bCompletionFreshChatAlways = true;
         this.bCompletionInsertStandardRolePrefix = false;
         this.bTrimGarbage = true;
@@ -659,6 +673,8 @@ class Me {
         let p = ui.el_create_append_p("Settings (devel-tools-console document[gMe])", elDiv);
         p.className = "role-system";
 
+        ui.el_create_append_p(`bStream:${this.bStream}`, elDiv);
+
         ui.el_create_append_p(`bCompletionFreshChatAlways:${this.bCompletionFreshChatAlways}`, elDiv);
 
         ui.el_create_append_p(`bCompletionInsertStandardRolePrefix:${this.bCompletionInsertStandardRolePrefix}`, elDiv);
@@ -679,7 +695,12 @@ class Me {
      */
     show_settings(elDiv) {
 
-        let bb = ui.el_creatediv_boolbutton("SetCompletionFreshChatAlways", "CompletionFreshChatAlways", {true: "[+] yes fresh", false: "[-] no, with history"}, this.bCompletionFreshChatAlways, (val)=>{
+        let bb = ui.el_creatediv_boolbutton("SetStream", "Stream", {true: "[+] yes stream", false: "[-] do oneshot"}, this.bStream, (val)=>{
+            this.bStream = val;
+        });
+        elDiv.appendChild(bb);
+
+        bb = ui.el_creatediv_boolbutton("SetCompletionFreshChatAlways", "CompletionFreshChatAlways", {true: "[+] yes fresh", false: "[-] no, with history"}, this.bCompletionFreshChatAlways, (val)=>{
             this.bCompletionFreshChatAlways = val;
         });
         elDiv.appendChild(bb);