From f33aa28149de423b6bf7cf722046a1f2046f5f08 Mon Sep 17 00:00:00 2001
From: HanishKVC <hanishkvc@gmail.com>
Date: Sun, 26 May 2024 01:07:07 +0530
Subject: [PATCH] SimpleChat:DU: Try trim using histogram based info

TODO: May have to add max number of uniq chars in histogram at
end of learning phase.
---
 .../server/public_simplechat/datautils.mjs    | 52 +++++++++++++++++++
 .../server/public_simplechat/simplechat.js    |  3 +-
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/examples/server/public_simplechat/datautils.mjs b/examples/server/public_simplechat/datautils.mjs
index 8d513d8b1..64ffcf745 100644
--- a/examples/server/public_simplechat/datautils.mjs
+++ b/examples/server/public_simplechat/datautils.mjs
@@ -68,3 +68,55 @@ export function trim_repeat_garbage_at_end_loop(sIn, maxSubL, maxMatchLenThresho
         sCur = got.data;
     }
 }
+
+
+/**
+ * A simple minded try trim garbage at end using histogram characteristics
+ * @param {string} sIn
+ * @param {number} maxSubL
+ * @param {number} maxMatchLenThreshold
+ */
+export function trim_hist_garbage_at_end(sIn, maxSubL, maxMatchLenThreshold) {
+    if (sIn.length < maxMatchLenThreshold) {
+        return { trimmed: false, data: sIn };
+    }
+    // Learn
+    let hist = {};
+    for(let i=0; i<maxSubL; i++) {
+        let c = sIn[sIn.length-1-i];
+        if (c in hist) {
+            hist[c] += 1;
+        } else {
+            hist[c] = 1;
+        }
+    }
+    console.log("DBUG:TrimHistGarbage:", hist);
+    // Catch and Trim
+    for(let i=0; i < sIn.length; i++) {
+        let c = sIn[sIn.length-1-i];
+        if (!(c in hist)) {
+            if (i < maxMatchLenThreshold) {
+                return { trimmed: false, data: sIn };
+            }
+            return { trimmed: true, data: sIn.substring(0, sIn.length-i+1) };
+        }
+    }
+    return { trimmed: true, data: "" };
+}
+
+/**
+ * Keep trimming repeatedly using hist_garbage logic, till you no longer can
+ * @param {any} sIn
+ * @param {number} maxSubL
+ * @param {number} maxMatchLenThreshold
+ */
+export function trim_hist_garbage_at_end_loop(sIn, maxSubL, maxMatchLenThreshold) {
+    let sCur = sIn;
+    while (true) {
+        let got = trim_hist_garbage_at_end(sCur, maxSubL, maxMatchLenThreshold);
+        if (!got.trimmed) {
+            return got.data;
+        }
+        sCur = got.data;
+    }
+}
diff --git a/examples/server/public_simplechat/simplechat.js b/examples/server/public_simplechat/simplechat.js
index 8313c4071..c09ca73b1 100644
--- a/examples/server/public_simplechat/simplechat.js
+++ b/examples/server/public_simplechat/simplechat.js
@@ -481,7 +481,7 @@ class MultiChatUI {
                 assistantMsg = respBody["content"];
             }
         }
-        assistantMsg = du.trim_repeat_garbage_at_end_loop(assistantMsg, 32, 72);
+        assistantMsg = du.trim_hist_garbage_at_end_loop(assistantMsg, 12, 72);
         chat.add(Roles.Assistant, assistantMsg);
         if (chatId == this.curChatId) {
             chat.show(this.elDivChat);
@@ -623,6 +623,7 @@ function startme() {
     console.log("INFO:SimpleChat:StartMe:Starting...");
     gMe = new Me();
     document["gMe"] = gMe;
+    document["du"] = du;
     for (let cid of gMe.defaultChatIds) {
         gMe.multiChat.new_chat_session(cid);
     }