vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pass vocab to common_chat_params_init

Browse source
This commit is contained in:
ochafik 2025-02-03 01:16:02 +00:00
parent d3b60b8ad8
commit 87de852b7f
4 changed files with 11 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

8
examples/server/server.cpp
View file

@ -1907,9 +1907,9 @@ struct server_context {
}});
GGML_ASSERT(templates.template_default);
try {
common_chat_params_init(*templates.template_default, inputs);
common_chat_params_init(*templates.template_default, inputs, vocab);
if (templates.template_tool_use) {
common_chat_params_init(*templates.template_tool_use, inputs);
common_chat_params_init(*templates.template_tool_use, inputs, vocab);
}
return true;
} catch (const std::exception & e) {
@ -4048,7 +4048,7 @@ int main(int argc, char ** argv) {
}
auto body = json::parse(req.body);
json data = oaicompat_completion_params_parse(body, params.use_jinja, ctx_server.chat_templates);
json data = oaicompat_completion_params_parse(body, params.use_jinja, ctx_server.chat_templates, llama_model_get_vocab(ctx_server.model));
return handle_completions_impl(
SERVER_TASK_TYPE_COMPLETION,
@ -4061,7 +4061,7 @@ int main(int argc, char ** argv) {
// same with handle_chat_completions, but without inference part
const auto handle_apply_template = [&ctx_server, &params, &res_ok](const httplib::Request & req, httplib::Response & res) {
auto body = json::parse(req.body);
json data = oaicompat_completion_params_parse(body, params.use_jinja, ctx_server.chat_templates);
json data = oaicompat_completion_params_parse(body, params.use_jinja, ctx_server.chat_templates, llama_model_get_vocab(ctx_server.model));
res_ok(res, {{ "prompt", std::move(data.at("prompt")) }});
};

5
examples/server/utils.hpp
View file

@ -582,7 +582,8 @@ static json oaicompat_completion_params_parse(const json & body) {
static json oaicompat_completion_params_parse(
const json & body, /* openai api json semantics */
bool use_jinja,
const common_chat_templates & chat_templates)
const common_chat_templates & chat_templates,
const llama_vocab * vocab)
{
json llama_params;
const auto & tmpl = body.contains("tools") && chat_templates.template_tool_use
@ -648,7 +649,7 @@ static json oaicompat_completion_params_parse(
inputs.stream = stream;
// TODO: support mixing schema w/ tools beyond generic format.
inputs.json_schema = json_value(llama_params, "json_schema", json());
auto chat_params = common_chat_params_init(tmpl, inputs);
auto chat_params = common_chat_params_init(tmpl, inputs, vocab);
llama_params["chat_format"] = static_cast<int>(chat_params.format);
llama_params["prompt"] = chat_params.prompt;