export llama_timings as struct and expose them in server

This commit is contained in:
Tobias Lütke 2023-07-04 21:52:04 -04:00
parent c19daa4eb5
commit efa86bf2a6
No known key found for this signature in database
GPG key ID: 1FC0DBB14164709A
7 changed files with 1170 additions and 1001 deletions

View file

@ -5,6 +5,8 @@ const paramDefaults = {
stop: ["</s>"]
};
let generation_settings = null;
/**
* This function completes the input text using a llama dictionary.
* @param {object} params - The parameters for the completion request.
@ -66,6 +68,9 @@ export const llamaComplete = async (params, controller, callback) => {
// if we got a stop token from server, we will break here
if (result.data.stop) {
if(result.data.generation_settings) {
generation_settings = result.data.generation_settings;
}
break;
}
}
@ -79,3 +84,11 @@ export const llamaComplete = async (params, controller, callback) => {
return content;
}
export const llamaModelInfo = async () => {
if (!generation_settings) {
generation_settings = await fetch("/model.json").then(r => r.json());
}
return generation_settings;
}