vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tool-call: fix functionary-small-3.2 (first tool starts w/ name\n, subsequent are >>>name\n)

Browse source
This commit is contained in:
ochafik 2024-09-27 07:06:33 +01:00
parent e33b342da7
commit e62b5de3cf
3 changed files with 50 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

47
common/tool-call.cpp
View file

@ -133,13 +133,20 @@ static llama_tool_calls parse_llama_3_1_tool_calls(const json & tools, const std
return {input, {}}; return {input, {}};
} }
static llama_tool_calls parse_functionary_tool_calls(const std::string& input, const std::regex & function_regex, const std::regex & close_regex) { static llama_tool_calls parse_functionary_tool_calls(const json & tools, const std::string& input, const std::regex & function_regex, const std::regex & close_regex) {
std::smatch match; std::smatch match;
llama_tool_calls result; llama_tool_calls result;
auto end = input.end(); auto end = input.end();
auto it = input.begin(); auto it = input.begin();
std::unordered_set<std::string> tool_names;
for (const auto & tool : tools) {
if (tool.contains("type") && tool["type"] == "function") {
tool_names.insert(tool["function"]["name"]);
}
}
while (it != end) { while (it != end) {
std::sregex_iterator rend; std::sregex_iterator rend;
std::sregex_iterator rit(it, end, function_regex); std::sregex_iterator rit(it, end, function_regex);
@ -147,11 +154,15 @@ static llama_tool_calls parse_functionary_tool_calls(const std::string& input, c
result.content += std::string(it, end); result.content += std::string(it, end);
break; break;
} }
auto name = rit->str(1);
if (tool_names.find(name) == tool_names.end()) {
result.content += std::string(it, rit->suffix().first);
break;
}
result.content += std::string(it, rit->prefix().second); result.content += std::string(it, rit->prefix().second);
it = rit->suffix().first; it = rit->suffix().first;
auto name = rit->str(1);
json arguments; json arguments;
if (!parse_json(it, end, arguments)) { if (!parse_json(it, end, arguments)) {
@ -166,7 +177,7 @@ static llama_tool_calls parse_functionary_tool_calls(const std::string& input, c
return result; return result;
} }
static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) { static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const json & tools, const std::string& input) {
// This version of Functionary still supports the llama 3.1 tool call format for the python tool. // This version of Functionary still supports the llama 3.1 tool call format for the python tool.
static std::regex python_tag_regex(R"(<\|python_tag\|>([\s\S\n]*)$)"); static std::regex python_tag_regex(R"(<\|python_tag\|>([\s\S\n]*)$)");
std::smatch match; std::smatch match;
@ -179,13 +190,13 @@ static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::str
} }
static std::regex function_regex(R"(<function=(\w+)>)"); static std::regex function_regex(R"(<function=(\w+)>)");
static std::regex close_regex(R"(</function>)"); static std::regex close_regex(R"(</function>)");
return parse_functionary_tool_calls(input, function_regex, close_regex); return parse_functionary_tool_calls(tools, input, function_regex, close_regex);
} }
static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input) { static llama_tool_calls parse_functionary_v3_tool_calls(const json & tools, const std::string& input) {
static std::regex function_regex(R"(>>>(\w+)\n)"); static std::regex function_regex(R"((?:>>>)?(\w+)\n)");
static std::regex close_regex(R"($|\n(?=>>>))"); static std::regex close_regex(R"($|\n(?=>>>))");
return parse_functionary_tool_calls(input, function_regex, close_regex); return parse_functionary_tool_calls(tools, input, function_regex, close_regex);
} }
llama_tool_calls parse_tool_calls(llama_tool_call_style style, const json & tools, const std::string& input) { llama_tool_calls parse_tool_calls(llama_tool_call_style style, const json & tools, const std::string& input) {
@ -193,9 +204,9 @@ llama_tool_calls parse_tool_calls(llama_tool_call_style style, const json & tool
case llama_tool_call_style::Llama31: case llama_tool_call_style::Llama31:
return parse_llama_3_1_tool_calls(tools, input); return parse_llama_3_1_tool_calls(tools, input);
case llama_tool_call_style::FunctionaryV3Llama3: case llama_tool_call_style::FunctionaryV3Llama3:
return parse_functionary_v3_tool_calls(input); return parse_functionary_v3_tool_calls(tools, input);
case llama_tool_call_style::FunctionaryV3Llama31: case llama_tool_call_style::FunctionaryV3Llama31:
return parse_functionary_v3_llama_3_1_tool_calls(input); return parse_functionary_v3_llama_3_1_tool_calls(tools, input);
case llama_tool_call_style::Hermes2Pro: case llama_tool_call_style::Hermes2Pro:
return parse_hermes_tool_calls(input); return parse_hermes_tool_calls(input);
default: default:
@ -250,20 +261,28 @@ llama_tool_call_handler llama_tool_call_handler_init(
// >>>all\nlet's call functions>>>fn1\n{"arg1": 1...}\n>>>fn2\n{"arg1": 1...}... // >>>all\nlet's call functions>>>fn1\n{"arg1": 1...}\n>>>fn2\n{"arg1": 1...}...
// Using ">>>f1\n", ">>>f2\n"... as trigger words for the grammar // Using ">>>f1\n", ">>>f2\n"... as trigger words for the grammar
handler.grammar = build_grammar([&](const llama_grammar_builder & builder) { handler.grammar = build_grammar([&](const llama_grammar_builder & builder) {
std::vector<std::string> tool_rules; std::vector<std::string> first_tool_rules;
std::vector<std::string> subsequent_tool_rules;
for (size_t i = 0, n = tools.size(); i < n; i++) { for (size_t i = 0, n = tools.size(); i < n; i++) {
auto & tool = tools[i]; auto & tool = tools[i];
const auto & function = tool["function"]; const auto & function = tool["function"];
std::string name = function["name"]; std::string name = function["name"];
auto parameters = function["parameters"]; auto parameters = function["parameters"];
auto tool_rule = builder.add_rule(name + "-call", "\">>>" + name + "\\n\" " + builder.add_schema(name + "-args", parameters)); auto args_rule = builder.add_schema(name + "-args", parameters);
tool_rules.push_back(tool_rule); first_tool_rules.push_back(builder.add_rule(name + "-call", "\"" + name + "\\n\" " + args_rule));
subsequent_tool_rules.push_back(builder.add_rule(name + "-call2", "\"\\n>>>" + name + "\\n\" " + args_rule));
if (allow_content) { if (allow_content) {
handler.grammar_trigger_words.push_back(name + "\n");
handler.grammar_trigger_words.push_back(">>>" + name + "\n"); handler.grammar_trigger_words.push_back(">>>" + name + "\n");
} }
} }
auto tool_call = builder.add_rule("tool_call", join(tool_rules.begin(), tool_rules.end(), " | ")) + " space"; auto first_rule = builder.add_rule("first_tool_call", join(first_tool_rules.begin(), first_tool_rules.end(), " | ")) + " space";
builder.add_rule("root", parallel_tool_calls ? "(" + tool_call + ")+" : tool_call); if (parallel_tool_calls) {
auto subsequent_rule = builder.add_rule("subsequent_tool_call", join(subsequent_tool_rules.begin(), subsequent_tool_rules.end(), " | ")) + " space";
builder.add_rule("root", first_rule + " (" + subsequent_rule + ")*");
} else {
builder.add_rule("root", first_rule);
}
}); });
// handler.parser = parse_functionary_3_2_tool_calls; // handler.parser = parse_functionary_3_2_tool_calls;
break; break;

19
examples/agent/README.md
View file

@ -7,11 +7,24 @@
```bash ```bash
make -j LLAMA_CURL=1 llama-server make -j LLAMA_CURL=1 llama-server
./llama-server \ ./llama-server \
-mu https://huggingface.co/lmstudio-community/Meta-Llama-3.1-70B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-70B-Instruct-Q4_K_M.gguf \ --jinja -fa \
--jinja \ -mu https://huggingface.co/lmstudio-community/Meta-Llama-3.1-70B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-70B-Instruct-Q4_K_M.gguf
-c 8192 -fa
``` ```
<details>
<summary>Instructions for meekai/functionary-small-v3.2 (experimental)</summary>
The template in the GGUF doesn't seem to support tool calls, but its bigger brother's template can be used:
```bash
./llama-server \
--jinja -fa \
-mu https://huggingface.co/meetkai/functionary-small-v3.2-GGUF/resolve/main/functionary-small-v3.2.Q4_0.gguf \
--chat-template-file tests/chat/templates/meetkai-functionary-medium-v3.2.jinja
```
</details>
- Run some tools inside a docker container (check http://localhost:8088/docs once running): - Run some tools inside a docker container (check http://localhost:8088/docs once running):
```bash ```bash

1
examples/agent/fastify.py
View file

@ -4,6 +4,7 @@
# "fastapi", # "fastapi",
# "uvicorn", # "uvicorn",
# "typer", # "typer",
# "ipython",
# ] # ]
# /// # ///
''' '''