vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

javascript chat update.

Browse source
This commit is contained in:
Henri Vasserman 2023-06-12 18:34:01 +03:00
parent 13cf6929b7
commit b91200a2e5
No known key found for this signature in database
GPG key ID: 2995FC0F58B1A986
2 changed files with 72 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/server/README.md
View file

@ -167,7 +167,7 @@ node .
### Interactive mode ### Interactive mode
Check the sample in [chat.mjs](chat.mjs). Check the sample in [chat.mjs](chat.mjs).
Run with node: Run with NodeJS version 16 or later:
```sh ```sh
node chat.mjs node chat.mjs

68
examples/server/chat.mjs
View file

@ -1,21 +1,41 @@
import * as readline from 'node:readline/promises'; import * as readline from 'node:readline'
import { stdin as input, stdout as output } from 'node:process'; import { stdin, stdout } from 'node:process'
const chat = [ const chat = [
{ human: "Hello, Assistant.", {
assistant: "Hello. How may I help you today?" }, human: "Hello, Assistant.",
{ human: "Please tell me the largest city in Europe.", assistant: "Hello. How may I help you today?"
assistant: "Sure. The largest city in Europe is Moscow, the capital of Russia." }, },
{
human: "Please tell me the largest city in Europe.",
assistant: "Sure. The largest city in Europe is Moscow, the capital of Russia."
},
] ]
const instruction = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.`
function format_prompt(question) { function format_prompt(question) {
return "A chat between a curious human and an artificial intelligence assistant. " return `${instruction}\n${
+ "The assistant gives helpful, detailed, and polite answers to the human's questions.\n" chat.map(m =>`### Human: ${m.human}\n### Assistant: ${m.assistant}`).join("\n")
+ chat.map(m => `### Human: ${m.human}\n### Assistant: ${m.assistant}`).join("\n") }\n### Human: ${question}\n### Assistant:`
+ `\n### Human: ${question}\n### Assistant:`
} }
async function ChatCompletion(question) { async function tokenize(content) {
const result = await fetch("http://127.0.0.1:8080/tokenize", {
method: 'POST',
body: JSON.stringify({ content })
})
if (!result.ok) {
return []
}
return await result.json().tokens
}
const n_keep = await tokenize(instruction).length
async function chat_completion(question) {
const result = await fetch("http://127.0.0.1:8080/completion", { const result = await fetch("http://127.0.0.1:8080/completion", {
method: 'POST', method: 'POST',
body: JSON.stringify({ body: JSON.stringify({
@ -23,7 +43,7 @@ async function ChatCompletion(question) {
temperature: 0.2, temperature: 0.2,
top_k: 40, top_k: 40,
top_p: 0.9, top_p: 0.9,
n_keep: 29, n_keep: n_keep,
n_predict: 256, n_predict: 256,
stop: ["\n### Human:"], // stop completion after generating this stop: ["\n### Human:"], // stop completion after generating this
stream: true, stream: true,
@ -31,7 +51,7 @@ async function ChatCompletion(question) {
}) })
if (!result.ok) { if (!result.ok) {
return; return
} }
let answer = '' let answer = ''
@ -42,20 +62,26 @@ async function ChatCompletion(question) {
const message = JSON.parse(t.substring(6)) const message = JSON.parse(t.substring(6))
answer += message.content answer += message.content
process.stdout.write(message.content) process.stdout.write(message.content)
if (message.stop) break; if (message.stop) {
if (message.truncated) {
chat.shift()
}
break
}
} }
} }
process.stdout.write('\n') process.stdout.write('\n')
chat.push({ human: question, assistant: answer }) chat.push({ human: question, assistant: answer.trimStart() })
} }
const rl = readline.createInterface({ input, output }); const rl = readline.createInterface({ input: stdin, output: stdout });
const readlineQuestion = (rl, query, options) => new Promise((resolve, reject) => {
rl.question(query, options, resolve)
});
while(true) { while(true) {
const question = await readlineQuestion(rl, '> ')
const question = await rl.question('> ') await chat_completion(question)
await ChatCompletion(question);
} }