vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama 3.1: allow {name: & {function: syntax even w/ builtin tools (70B model just likes that!)

Browse source
This commit is contained in:
ochafik 2025-01-30 00:43:06 +00:00
parent d86a1ae80d
commit 774557cfb4
1 changed files with 5 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

13
common/chat.cpp
View file

@ -384,12 +384,12 @@ static common_chat_params common_chat_params_init_llama_3_1_tool_calls(const com
const auto & function = tool["function"]; const auto & function = tool["function"];
std::string name = function["name"]; std::string name = function["name"];
auto parameters = function["parameters"]; auto parameters = function["parameters"];
builder.resolve_refs(parameters);
// https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/remote/tool_runtime // https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/remote/tool_runtime
if (allow_python_tag_builtin_tools && handle_builtin_tool(name, parameters)) { if (allow_python_tag_builtin_tools) {
return; handle_builtin_tool(name, parameters);
} }
builder.resolve_refs(parameters);
tool_rules.push_back( tool_rules.push_back(
builder.add_rule( builder.add_rule(
name + "-call", name + "-call",
@ -398,12 +398,9 @@ static common_chat_params common_chat_params_init_llama_3_1_tool_calls(const com
builder.add_schema(name + "-args", parameters) + builder.add_schema(name + "-args", parameters) +
" \"}\"")); " \"}\""));
data.grammar_triggers.push_back({"{\"name\": \"" + name + "\"", /* .at_start = */ true}); data.grammar_triggers.push_back({"{\"name\": \"" + name + "\"", /* .at_start = */ true});
has_function = true;
}); });
if (has_function) { data.grammar_triggers.push_back({"{\"name\":", /* .at_start = */ true});
data.grammar_triggers.push_back({"{\"name\":", /* .at_start = */ true}); data.grammar_triggers.push_back({"{\"type\": \"function\"", /* .at_start = */ true});
data.grammar_triggers.push_back({"{\"type\": \"function\"", /* .at_start = */ true});
}
if (!builtin_tools.empty()) { if (!builtin_tools.empty()) {
data.grammar_triggers.push_back({"<|python_tag|>", /* .at_start = */ false}); data.grammar_triggers.push_back({"<|python_tag|>", /* .at_start = */ false});
} }