vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tool-call: fix/test functionary v3

Browse source
This commit is contained in:
ochafik 2024-09-26 03:42:05 +01:00
parent c124ab48ea
commit 595e11cb11
4 changed files with 40 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

32
common/tool-call.cpp
View file

@ -39,6 +39,8 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
std::size_t position; std::size_t position;
bool found_error; bool found_error;
json_error_locator() : position(0), found_error(false) {}
bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override { bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override {
// LOG_WARNING("JSON error (Expected)", {{"position", position}, {"last_token", last_token}, {"error", ex.what()}}); // LOG_WARNING("JSON error (Expected)", {{"position", position}, {"last_token", last_token}, {"error", ex.what()}});
this->position = position - 1; this->position = position - 1;
@ -67,7 +69,7 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
} else { } else {
temptative_end = end; temptative_end = end;
} }
std::string json_sub {it, it + err_loc.position}; std::string json_sub {it, temptative_end};
// LOG_WARNING("Parsing json", {{"json_sub", json_sub}}); // LOG_WARNING("Parsing json", {{"json_sub", json_sub}});
try { try {
out = json::parse(json_sub); out = json::parse(json_sub);
@ -155,9 +157,7 @@ static llama_tool_calls parse_llama_3_1_tool_calls(const json & tools, const std
return {input, {}}; return {input, {}};
} }
static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) { static llama_tool_calls parse_functionary_tool_calls(const std::string& input, const std::regex & function_regex, const std::regex & close_regex) {
static std::regex function_regex(R"(<function=(\w+)>)");
static std::regex close_regex(R"(</function>)");
std::smatch match; std::smatch match;
llama_tool_calls result; llama_tool_calls result;
@ -190,22 +190,16 @@ static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::str
return result; return result;
} }
static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input) { static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
static std::regex python_tag_regex(R"(>>>(\w+)\n((?!>>>)[\s\S\n]*))"); static std::regex function_regex(R"(<function=(\w+)>)");
std::smatch match; static std::regex close_regex(R"(</function>)");
llama_tool_calls result; return parse_functionary_tool_calls(input, function_regex, close_regex);
std::string content;
std::string in = input;
while (std::regex_search(in, match, python_tag_regex)) {
content += match.prefix().str();
result.tool_calls.push_back({
match[1].str(),
(json {{"code", match[2].str()}}).dump(),
});
in = match.suffix().str();
} }
result.content = content + in;
return result; static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input) {
static std::regex function_regex(R"(>>>(\w+)\n)");
static std::regex close_regex(R"($|\n(?=>>>))");
return parse_functionary_tool_calls(input, function_regex, close_regex);
} }
llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) { llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) {

2
examples/server/tests/features/steps/steps.py
View file

@ -166,7 +166,7 @@ def step_use_jinja(context):
context.use_jinja = True context.use_jinja = True
@step('chat template file {file}') @step('a chat template file {file}')
def step_use_jinja(context, file): def step_use_jinja(context, file):
context.chat_template_file = file context.chat_template_file = file

30
examples/server/tests/features/tool_call.feature
View file

@ -15,34 +15,36 @@ Feature: llama.cpp server
And 64 server max tokens to predict And 64 server max tokens to predict
And prometheus compatible metrics exposed And prometheus compatible metrics exposed
And jinja templates are enabled And jinja templates are enabled
And chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
Then the server is starting
Then the server is healthy
Scenario: Health
Then the server is ready
And all slots are idle
@wip
Scenario Outline: OAI Compatibility w/ required tool Scenario Outline: OAI Compatibility w/ required tool
Given a model test Given a chat template file ../../../tests/chat/templates/<template_name>.jinja
And the server is starting
And the server is healthy
And a model test
And <n> max tokens to predict And <n> max tokens to predict
And a user prompt write a hello world in python And a user prompt write a hello world in python
And a tool choice <tool_choice> And a tool choice <tool_choice>
And tools <tools> And tools <tools>
Given an OAI compatible chat completions request with no api error And an OAI compatible chat completions request with no api error
Then tool <tool_name> is called with arguments <tool_arguments> Then tool <tool_name> is called with arguments <tool_arguments>
Examples: Prompts Examples: Prompts
| n | tool_name | tool_arguments | tool_choice | tools | | template_name | n | tool_name | tool_arguments | tool_choice | tools |
| 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meetkai-functionary-medium-v3.2 | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.2 | 64 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
Scenario: OAI Compatibility w/ no tool Scenario: OAI Compatibility w/ no tool
Given a model test Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
And the server is starting
And the server is healthy
And a model test
And 16 max tokens to predict And 16 max tokens to predict
And a user prompt write a hello world in python And a user prompt write a hello world in python
And a tool choice <tool_choice> And a tool choice <tool_choice>
And tools [] And tools []
Given an OAI compatible chat completions request with no api error And an OAI compatible chat completions request with no api error
Then no tool is called Then no tool is called

11
tests/test-tool-call.cpp
View file

@ -74,7 +74,7 @@ int main() {
std::string functionary_v3_like_tmpl = "Functionary 3.2 template should have <|start_header_id|> and then some >>>all inside it"; std::string functionary_v3_like_tmpl = "Functionary 3.2 template should have <|start_header_id|> and then some >>>all inside it";
test_parse_tool_call(tools, functionary_v3_like_tmpl, test_parse_tool_call(tools, functionary_v3_like_tmpl,
">>>ipython\nprint('Hello, world!')", ">>>ipython\n{\"code\": \"print('Hello, world!')\"}",
"", "",
json {{ json {{
{"function", { {"function", {
@ -84,6 +84,15 @@ int main() {
}).dump()} }).dump()}
}} }}
}}); }});
test_parse_tool_call(tools, functionary_v3_like_tmpl,
">>>test\n{ } \n ",
"",
json {{
{"function", {
{"name", "test"},
{"arguments", "{}"}
}}
}});
std::string functionary_v3_llama_3_1_like_tmpl = "Functionary 3.2 template for llama 3.1 should have <|start_header_id|> and then some <function=foo>{...}</function> inside it"; std::string functionary_v3_llama_3_1_like_tmpl = "Functionary 3.2 template for llama 3.1 should have <|start_header_id|> and then some <function=foo>{...}</function> inside it";
test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl, test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl,