vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tool-call: only match json eagerly for Llama 3.2

Browse source
This commit is contained in:
ochafik 2024-09-28 19:05:10 +01:00
parent 6e0053a81b
commit 05bbba9f8a
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
common/tool-call.cpp
View file

@ -223,10 +223,10 @@ llama_tool_call_handler llama_tool_call_handler_init(
auto uses_python_tag = tmpl.tool_call_style() == llama_tool_call_style::Llama31; auto uses_python_tag = tmpl.tool_call_style() == llama_tool_call_style::Llama31;
// Technically we should only trigger on `"\n{\"name\": \"" + name + "\""` for each tool name, // Technically we should only trigger on `"\n{\"name\": \"" + name + "\""` for each tool name,
// but Llama-3.2-3B struggles to output valid tool calls so we're "guiding" it strongly as soon // but Llama-3.2-3B (and 1B) struggles to output valid tool calls so we're "guiding" it strongly as soon
// as it seems to be outputting some JSON. // as it seems to be outputting some JSON.
// TODO: make this conditional on a very small model (e.g. 1B / 3B). // TODO: make this conditional on a very small model (e.g. 1B / 3B).
auto eagerly_match_any_json = true; auto eagerly_match_any_json = tmpl.tool_call_style() == llama_tool_call_style::Llama32;
handler.grammar = build_grammar([&](const llama_grammar_builder & builder) { handler.grammar = build_grammar([&](const llama_grammar_builder & builder) {
std::vector<std::string> tool_rules; std::vector<std::string> tool_rules;