vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use javascript generators as much cleaner API

Browse source 
Also add ways to access completion as promise and EventSource
This commit is contained in:
Tobias Lütke 2023-07-05 15:03:01 -04:00
parent 983b555e9d
commit a76ce02a6c
No known key found for this signature in database
GPG key ID: 1FC0DBB14164709A
2 changed files with 106 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

12
examples/server/public/index.html
View file

@ -106,7 +106,7 @@
html, h, signal, effect, computed, render, useSignal, useEffect, useRef
} from '/index.js';
import { llamaComplete } from '/completion.js';
import { llama } from '/completion.js';
const session = signal({
prompt: "This is a conversation between user and llama, a friendly chatbot. respond in markdown.",
@ -158,7 +158,7 @@
transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
const payload = template(session.value.template, {
const prompt = template(session.value.template, {
message: msg,
history: session.value.transcript.flatMap(([name, message]) => template(session.value.historyTemplate, {name, message})).join("\n"),
});
@ -168,13 +168,13 @@
const llamaParams = {
...params.value,
prompt: payload,
stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
}
await llamaComplete(llamaParams, controller.value, (message) => {
const data = message.data;
for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) {
const data = chunk.data;
currentMessage += data.content;
// remove leading whitespace
currentMessage = currentMessage.replace(/^\s+/, "")
@ -183,7 +183,7 @@
if (data.stop) {
console.log("-->", data, ' response was:', currentMessage, 'transcript state:', session.value.transcript);
}
})
}
controller.value = null;
}