vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tool-call: fix passing of tools to template + allow agent to finish

Browse source
This commit is contained in:
ochafik 2024-09-27 06:24:22 +01:00
parent f62e688387
commit e33b342da7
8 changed files with 8 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

1
common/tool-call.h
View file

@ -21,7 +21,6 @@ struct llama_tool_call_handler {
std::string grammar; std::string grammar;
std::vector<std::string> grammar_trigger_words; std::vector<std::string> grammar_trigger_words;
std::vector<std::string> additional_stop_words; std::vector<std::string> additional_stop_words;
nlohmann::ordered_json updated_tools;
}; };
llama_tool_calls parse_tool_calls(llama_tool_call_style style, const nlohmann::ordered_json & tools, const std::string& input); llama_tool_calls parse_tool_calls(llama_tool_call_style style, const nlohmann::ordered_json & tools, const std::string& input);

3
examples/agent/README.md
View file

@ -15,8 +15,7 @@
- Run some tools inside a docker container (check http://localhost:8088/docs once running): - Run some tools inside a docker container (check http://localhost:8088/docs once running):
```bash ```bash
docker run -p 8088:8088 -w /src \ docker run -p 8088:8088 -w /src -v $PWD/examples/agent:/src \
-v $PWD/examples/agent:/src \
--rm -it ghcr.io/astral-sh/uv:python3.12-alpine \ --rm -it ghcr.io/astral-sh/uv:python3.12-alpine \
uv run fastify.py --port 8088 tools.py uv run fastify.py --port 8088 tools.py
``` ```

1
examples/agent/run.py
View file

@ -182,6 +182,7 @@ def main(
else: else:
assert content assert content
print(content) print(content)
return
i += 1 i += 1

1
examples/server/server.cpp
View file

@ -986,6 +986,7 @@ struct server_context {
{ {
slot.antiprompts.clear(); slot.antiprompts.clear();
slot.sparams.grammar_trigger_words.clear();
auto copy_string_array = [&](const json & data, const std::string & key, std::vector<std::string> & vec) { auto copy_string_array = [&](const json & data, const std::string & key, std::vector<std::string> & vec) {
const auto & arr = data.find(key); const auto & arr = data.find(key);

4
examples/server/tests/features/tool_call.feature
View file

@ -31,11 +31,11 @@ Feature: llama.cpp server
Examples: Prompts Examples: Prompts
| template_name | n_predict | tool_name | tool_arguments | tools | | template_name | n_predict | tool_name | tool_arguments | tools |
| meetkai-functionary-medium-v3.1 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | | meetkai-functionary-medium-v3.1 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "I'm sorry,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | | meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "Yes, you can."} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meetkai-functionary-medium-v3.2 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | | meetkai-functionary-medium-v3.2 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | | meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": ". A"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
Scenario Outline: OAI Compatibility w/ tools and auto tool_choice Scenario Outline: OAI Compatibility w/ tools and auto tool_choice

3
examples/server/utils.hpp
View file

@ -384,9 +384,6 @@ static json oaicompat_completion_params_parse(
} }
llama_params["grammar_trigger_words"] = triggers; llama_params["grammar_trigger_words"] = triggers;
} }
if (handler.updated_tools.is_null()) {
tools = handler.updated_tools;
}
if (!handler.grammar.empty()) { if (!handler.grammar.empty()) {
if (llama_params.contains("grammar")) { if (llama_params.contains("grammar")) {
throw std::runtime_error("Cannot use custom grammar constraints with tools."); throw std::runtime_error("Cannot use custom grammar constraints with tools.");