vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

simplify handle_apply_template

Browse source
This commit is contained in:
Xuan Son Nguyen 2025-01-30 12:00:54 +01:00
parent 2d51c459c6
commit c88f4a798d
3 changed files with 14 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

18
examples/server/server.cpp
View file

@ -4016,8 +4016,7 @@ int main(int argc, char ** argv) {
} }
auto body = json::parse(req.body); auto body = json::parse(req.body);
const auto & chat_template = body.contains("tools") && ctx_server.chat_templates.template_tool_use ? *ctx_server.chat_templates.template_tool_use : *ctx_server.chat_templates.template_default; json data = oaicompat_completion_params_parse(body, params.use_jinja, ctx_server.chat_templates);
json data = oaicompat_completion_params_parse(body, chat_template, params.use_jinja);
return handle_completions_impl( return handle_completions_impl(
SERVER_TASK_TYPE_COMPLETION, SERVER_TASK_TYPE_COMPLETION,
@ -4027,6 +4026,13 @@ int main(int argc, char ** argv) {
OAICOMPAT_TYPE_CHAT); OAICOMPAT_TYPE_CHAT);
}; };
// same with handle_chat_completions, but without inference part
const auto handle_apply_template = [&ctx_server, &params, &res_ok](const httplib::Request & req, httplib::Response & res) {
auto body = json::parse(req.body);
json data = oaicompat_completion_params_parse(body, params.use_jinja, ctx_server.chat_templates);
res_ok(res, {{ "prompt", std::move(data.at("prompt")) }});
};
const auto handle_models = [&params, &ctx_server, &res_ok](const httplib::Request &, httplib::Response & res) { const auto handle_models = [&params, &ctx_server, &res_ok](const httplib::Request &, httplib::Response & res) {
json models = { json models = {
{"object", "list"}, {"object", "list"},
@ -4185,14 +4191,6 @@ int main(int argc, char ** argv) {
res_ok(res, root); res_ok(res, root);
}; };
const auto handle_apply_template = [&ctx_server, &params, &res_ok](const httplib::Request & req, httplib::Response & res) {
auto body = json::parse(req.body);
const auto & chat_template = body.contains("tools") && ctx_server.chat_templates.template_tool_use ? *ctx_server.chat_templates.template_tool_use : *ctx_server.chat_templates.template_default;
json data = oaicompat_completion_params_parse(body, chat_template, params.use_jinja);
res_ok(res, {{ "prompt", data.at("prompt") }});
};
const auto handle_embeddings = [&handle_embeddings_impl](const httplib::Request & req, httplib::Response & res) { const auto handle_embeddings = [&handle_embeddings_impl](const httplib::Request & req, httplib::Response & res) {
handle_embeddings_impl(req, res, OAICOMPAT_TYPE_NONE); handle_embeddings_impl(req, res, OAICOMPAT_TYPE_NONE);
}; };

2
examples/server/tests/unit/test_tool_call.py
View file

@ -71,7 +71,7 @@ def do_test_completion_with_required_tool_tiny(template_name: str, tool: dict, a
server.jinja = True server.jinja = True
server.n_predict = n_predict server.n_predict = n_predict
server.chat_template_file = f'../../../models/templates/{template_name}.jinja' server.chat_template_file = f'../../../models/templates/{template_name}.jinja'
server.start() server.start(timeout_seconds=TIMEOUT_SERVER_START)
res = server.make_request("POST", "/chat/completions", data={ res = server.make_request("POST", "/chat/completions", data={
"max_tokens": n_predict, "max_tokens": n_predict,
"messages": [ "messages": [

7
examples/server/utils.hpp
View file

@ -580,10 +580,13 @@ static json oaicompat_completion_params_parse(const json & body) {
static json oaicompat_completion_params_parse( static json oaicompat_completion_params_parse(
const json & body, /* openai api json semantics */ const json & body, /* openai api json semantics */
const common_chat_template & tmpl, bool use_jinja,
bool use_jinja) const common_chat_templates & chat_templates)
{ {
json llama_params; json llama_params;
const auto & tmpl = body.contains("tools") && chat_templates.template_tool_use
? *chat_templates.template_tool_use
: *chat_templates.template_default;
auto tools = json_value(body, "tools", json()); auto tools = json_value(body, "tools", json());
auto stream = json_value(body, "stream", false); auto stream = json_value(body, "stream", false);