vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tool-call: fix/test functionary v3

Browse source
This commit is contained in:
ochafik 2024-09-26 03:42:05 +01:00
parent c124ab48ea
commit 595e11cb11
4 changed files with 40 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

32
common/tool-call.cpp
View file

@ -39,6 +39,8 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
std::size_t position;
bool found_error;
json_error_locator() : position(0), found_error(false) {}
bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override {
// LOG_WARNING("JSON error (Expected)", {{"position", position}, {"last_token", last_token}, {"error", ex.what()}});
this->position = position - 1;
@ -67,7 +69,7 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
} else {
temptative_end = end;
}
std::string json_sub {it, it + err_loc.position};
std::string json_sub {it, temptative_end};
// LOG_WARNING("Parsing json", {{"json_sub", json_sub}});
try {
out = json::parse(json_sub);
@ -155,9 +157,7 @@ static llama_tool_calls parse_llama_3_1_tool_calls(const json & tools, const std
return {input, {}};
}
static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
static std::regex function_regex(R"(<function=(\w+)>)");
static std::regex close_regex(R"(</function>)");
static llama_tool_calls parse_functionary_tool_calls(const std::string& input, const std::regex & function_regex, const std::regex & close_regex) {
std::smatch match;
llama_tool_calls result;
@ -190,22 +190,16 @@ static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::str
return result;
}
static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
static std::regex function_regex(R"(<function=(\w+)>)");
static std::regex close_regex(R"(</function>)");
return parse_functionary_tool_calls(input, function_regex, close_regex);
}
static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input) {
static std::regex python_tag_regex(R"(>>>(\w+)\n((?!>>>)[\s\S\n]*))");
std::smatch match;
llama_tool_calls result;
std::string content;
std::string in = input;
while (std::regex_search(in, match, python_tag_regex)) {
content += match.prefix().str();
result.tool_calls.push_back({
match[1].str(),
(json {{"code", match[2].str()}}).dump(),
});
in = match.suffix().str();
}
result.content = content + in;
return result;
static std::regex function_regex(R"(>>>(\w+)\n)");
static std::regex close_regex(R"($|\n(?=>>>))");
return parse_functionary_tool_calls(input, function_regex, close_regex);
}
llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) {

2
examples/server/tests/features/steps/steps.py
View file

@ -166,7 +166,7 @@ def step_use_jinja(context):
context.use_jinja = True
@step('chat template file {file}')
@step('a chat template file {file}')
def step_use_jinja(context, file):
context.chat_template_file = file

30
examples/server/tests/features/tool_call.feature
View file

@ -15,34 +15,36 @@ Feature: llama.cpp server
And 64 server max tokens to predict
And prometheus compatible metrics exposed
And jinja templates are enabled
And chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
Then the server is starting
Then the server is healthy
Scenario: Health
Then the server is ready
And all slots are idle
@wip
Scenario Outline: OAI Compatibility w/ required tool
Given a model test
Given a chat template file ../../../tests/chat/templates/<template_name>.jinja
And the server is starting
And the server is healthy
And a model test
And <n> max tokens to predict
And a user prompt write a hello world in python
And a tool choice <tool_choice>
And tools <tools>
Given an OAI compatible chat completions request with no api error
And an OAI compatible chat completions request with no api error
Then tool <tool_name> is called with arguments <tool_arguments>
Examples: Prompts
| n | tool_name | tool_arguments | tool_choice | tools |
| 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| template_name | n | tool_name | tool_arguments | tool_choice | tools |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meetkai-functionary-medium-v3.2 | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.2 | 64 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
Scenario: OAI Compatibility w/ no tool
Given a model test
Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
And the server is starting
And the server is healthy
And a model test
And 16 max tokens to predict
And a user prompt write a hello world in python
And a tool choice <tool_choice>
And tools []
Given an OAI compatible chat completions request with no api error
And an OAI compatible chat completions request with no api error
Then no tool is called

11
tests/test-tool-call.cpp
View file

@ -74,7 +74,7 @@ int main() {
std::string functionary_v3_like_tmpl = "Functionary 3.2 template should have <|start_header_id|> and then some >>>all inside it";
test_parse_tool_call(tools, functionary_v3_like_tmpl,
">>>ipython\nprint('Hello, world!')",
">>>ipython\n{\"code\": \"print('Hello, world!')\"}",
"",
json {{
{"function", {
@ -84,6 +84,15 @@ int main() {
}).dump()}
}}
}});
test_parse_tool_call(tools, functionary_v3_like_tmpl,
">>>test\n{ } \n ",
"",
json {{
{"function", {
{"name", "test"},
{"arguments", "{}"}
}}
}});
std::string functionary_v3_llama_3_1_like_tmpl = "Functionary 3.2 template for llama 3.1 should have <|start_header_id|> and then some <function=foo>{...}</function> inside it";
test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl,