vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SimpleChat:DU:TrimGarbage if unable try skip char and retry

Browse source
This commit is contained in:
HanishKVC 2024-05-25 23:37:24 +05:30
parent 54802dc184
commit 6390f3489a
1 changed files with 14 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

16
examples/server/public_simplechat/datautils.mjs
View file

@ -42,16 +42,28 @@ export function trim_repeat_garbage_at_end(sIn, maxSubL=10, maxMatchLenThreshold
/** /**
* Simple minded logic to help remove repeating garbage at end of the string, till it cant. * Simple minded logic to help remove repeating garbage at end of the string, till it cant.
* If its not able to trim, then it will try to skip a char at end and then trim, a few times.
* @param {string} sIn * @param {string} sIn
* @param {number} maxSubL * @param {number} maxSubL
* @param {number | undefined} [maxMatchLenThreshold] * @param {number | undefined} [maxMatchLenThreshold]
*/ */
export function trim_repeat_garbage_at_end_loop(sIn, maxSubL, maxMatchLenThreshold) { export function trim_repeat_garbage_at_end_loop(sIn, maxSubL, maxMatchLenThreshold, skipMax=16) {
let sCur = sIn; let sCur = sIn;
let sSaved = "";
let iTry = 0;
while(true) { while(true) {
let got = trim_repeat_garbage_at_end(sCur, maxSubL, maxMatchLenThreshold); let got = trim_repeat_garbage_at_end(sCur, maxSubL, maxMatchLenThreshold);
if (got.trimmed != true) { if (got.trimmed != true) {
return got.data; if (iTry == 0) {
sSaved = got.data;
}
iTry += 1;
if (iTry >= skipMax) {
return sSaved;
}
got.data = got.data.substring(0,got.data.length-1);
} else {
iTry = 0;
} }
sCur = got.data; sCur = got.data;
} }