vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use javascript generators as much cleaner API

Browse source 
Also add ways to access completion as promise and EventSource
This commit is contained in:
Tobias Lütke 2023-07-05 15:03:01 -04:00
parent efa86bf2a6
commit ff6e39f138
No known key found for this signature in database
GPG key ID: 1FC0DBB14164709A
2 changed files with 100 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

12
examples/server/public/index.html
View file

@ -110,7 +110,7 @@
html, h, signal, effect, computed, render, useSignal, useEffect, useRef
} from '/index.js';
import { llamaComplete } from '/completion.js';
import { llama } from '/completion.js';
const session = signal({
prompt: "This is a conversation between user and llama, a friendly chatbot. respond in simple markdown.",
@ -163,7 +163,7 @@
transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
const payload = template(session.value.template, {
const prompt = template(session.value.template, {
message: msg,
history: session.value.transcript.flatMap(([name, message]) => template(session.value.historyTemplate, {name, message})).join("\n"),
});
@ -173,13 +173,13 @@
const llamaParams = {
...params.value,
prompt: payload,
stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
}
await llamaComplete(llamaParams, controller.value, (message) => {
const data = message.data;
for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) {
const data = chunk.data;
currentMessage += data.content;
// remove leading whitespace
currentMessage = currentMessage.replace(/^\s+/, "")
@ -192,7 +192,7 @@
if (data.timings) {
llamaStats.value = data.timings;
}
})
}
controller.value = null;
}