diff --git a/examples/server/README.md b/examples/server/README.md index 56399be29..8b53dc36e 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -167,7 +167,7 @@ node . ### Interactive mode Check the sample in [chat.mjs](chat.mjs). -Run with node: +Run with NodeJS version 16 or later: ```sh node chat.mjs diff --git a/examples/server/chat.mjs b/examples/server/chat.mjs index 349937e94..34ff90343 100644 --- a/examples/server/chat.mjs +++ b/examples/server/chat.mjs @@ -1,61 +1,87 @@ -import * as readline from 'node:readline/promises'; -import { stdin as input, stdout as output } from 'node:process'; +import * as readline from 'node:readline' +import { stdin, stdout } from 'node:process' const chat = [ - { human: "Hello, Assistant.", - assistant: "Hello. How may I help you today?" }, - { human: "Please tell me the largest city in Europe.", - assistant: "Sure. The largest city in Europe is Moscow, the capital of Russia." }, + { + human: "Hello, Assistant.", + assistant: "Hello. How may I help you today?" + }, + { + human: "Please tell me the largest city in Europe.", + assistant: "Sure. The largest city in Europe is Moscow, the capital of Russia." + }, ] +const instruction = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.` + function format_prompt(question) { - return "A chat between a curious human and an artificial intelligence assistant. " - + "The assistant gives helpful, detailed, and polite answers to the human's questions.\n" - + chat.map(m => `### Human: ${m.human}\n### Assistant: ${m.assistant}`).join("\n") - + `\n### Human: ${question}\n### Assistant:` + return `${instruction}\n${ + chat.map(m =>`### Human: ${m.human}\n### Assistant: ${m.assistant}`).join("\n") + }\n### Human: ${question}\n### Assistant:` } -async function ChatCompletion(question) { - const result = await fetch("http://127.0.0.1:8080/completion", { - method: 'POST', - body: JSON.stringify({ - prompt: format_prompt(question), - temperature: 0.2, - top_k: 40, - top_p: 0.9, - n_keep: 29, - n_predict: 256, - stop: ["\n### Human:"], // stop completion after generating this - stream: true, +async function tokenize(content) { + const result = await fetch("http://127.0.0.1:8080/tokenize", { + method: 'POST', + body: JSON.stringify({ content }) }) - }) - if (!result.ok) { - return; - } - - let answer = '' - - for await (var chunk of result.body) { - const t = Buffer.from(chunk).toString('utf8') - if (t.startsWith('data: ')) { - const message = JSON.parse(t.substring(6)) - answer += message.content - process.stdout.write(message.content) - if (message.stop) break; + if (!result.ok) { + return [] } - } - process.stdout.write('\n') - chat.push({ human: question, assistant: answer }) + return await result.json().tokens } -const rl = readline.createInterface({ input, output }); +const n_keep = await tokenize(instruction).length + +async function chat_completion(question) { + const result = await fetch("http://127.0.0.1:8080/completion", { + method: 'POST', + body: JSON.stringify({ + prompt: format_prompt(question), + temperature: 0.2, + top_k: 40, + top_p: 0.9, + n_keep: n_keep, + n_predict: 256, + stop: ["\n### Human:"], // stop completion after generating this + stream: true, + }) + }) + + if (!result.ok) { + return + } + + let answer = '' + + for await (var chunk of result.body) { + const t = Buffer.from(chunk).toString('utf8') + if (t.startsWith('data: ')) { + const message = JSON.parse(t.substring(6)) + answer += message.content + process.stdout.write(message.content) + if (message.stop) { + if (message.truncated) { + chat.shift() + } + break + } + } + } + + process.stdout.write('\n') + chat.push({ human: question, assistant: answer.trimStart() }) +} + +const rl = readline.createInterface({ input: stdin, output: stdout }); + +const readlineQuestion = (rl, query, options) => new Promise((resolve, reject) => { + rl.question(query, options, resolve) +}); while(true) { - - const question = await rl.question('> ') - await ChatCompletion(question); - + const question = await readlineQuestion(rl, '> ') + await chat_completion(question) } -