vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server : allow to get default generation settings for completion (#5307)

Browse source
This commit is contained in:
Alexey Parfenov 2024-02-05 08:10:22 +00:00 • committed by GitHub
parent e6f8177532
commit a2d60c9158
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 21 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
examples/server/server.cpp
View file

@ -334,6 +334,7 @@ struct llama_server_context
// slots / clients
std::vector<llama_client_slot> slots;
json default_generation_settings_for_props;
llama_server_queue queue_tasks;
llama_server_response queue_results;
@ -430,6 +431,9 @@ struct llama_server_context
slots.push_back(slot);
}
default_generation_settings_for_props = get_formated_generation(slots.front());
default_generation_settings_for_props["seed"] = -1;
batch = llama_batch_init(n_ctx, 0, params.n_parallel);
// empty system prompt
@ -2614,7 +2618,8 @@ int main(int argc, char **argv)
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
json data = {
{ "user_name", llama.name_user.c_str() },
{ "assistant_name", llama.name_assistant.c_str() }
{ "assistant_name", llama.name_assistant.c_str() },
{ "default_generation_settings", llama.default_generation_settings_for_props }
};
res.set_content(data.dump(), "application/json; charset=utf-8");
});