From e33b342da7058ad073bf346ee03b1243bd85acaf Mon Sep 17 00:00:00 2001 From: ochafik Date: Fri, 27 Sep 2024 06:24:22 +0100 Subject: [PATCH] `tool-call`: fix passing of tools to template + allow agent to finish --- common/tool-call.h | 1 - examples/agent/README.md | 3 +-- examples/agent/run.py | 1 + examples/server/server.cpp | 3 ++- examples/server/tests/features/tool_call.feature | 4 ++-- examples/server/utils.hpp | 3 --- src/llama-sampling.cpp | 2 +- src/llama.cpp | 2 +- 8 files changed, 8 insertions(+), 11 deletions(-) diff --git a/common/tool-call.h b/common/tool-call.h index 1cc9f8374..7c2af245c 100644 --- a/common/tool-call.h +++ b/common/tool-call.h @@ -21,7 +21,6 @@ struct llama_tool_call_handler { std::string grammar; std::vector grammar_trigger_words; std::vector additional_stop_words; - nlohmann::ordered_json updated_tools; }; llama_tool_calls parse_tool_calls(llama_tool_call_style style, const nlohmann::ordered_json & tools, const std::string& input); diff --git a/examples/agent/README.md b/examples/agent/README.md index fd5d37a71..f19cb5071 100644 --- a/examples/agent/README.md +++ b/examples/agent/README.md @@ -15,8 +15,7 @@ - Run some tools inside a docker container (check http://localhost:8088/docs once running): ```bash - docker run -p 8088:8088 -w /src \ - -v $PWD/examples/agent:/src \ + docker run -p 8088:8088 -w /src -v $PWD/examples/agent:/src \ --rm -it ghcr.io/astral-sh/uv:python3.12-alpine \ uv run fastify.py --port 8088 tools.py ``` diff --git a/examples/agent/run.py b/examples/agent/run.py index edccc5aa5..d811bca0f 100644 --- a/examples/agent/run.py +++ b/examples/agent/run.py @@ -182,6 +182,7 @@ def main( else: assert content print(content) + return i += 1 diff --git a/examples/server/server.cpp b/examples/server/server.cpp index cc509d286..4f7a29545 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -986,6 +986,7 @@ struct server_context { { slot.antiprompts.clear(); + slot.sparams.grammar_trigger_words.clear(); auto copy_string_array = [&](const json & data, const std::string & key, std::vector & vec) { const auto & arr = data.find(key); @@ -1004,7 +1005,7 @@ struct server_context { copy_string_array(data, "grammar_trigger_words", slot.sparams.grammar_trigger_words); slot.antiprompts.build(ctx, stop_words, slot.sparams.grammar_trigger_words); - + } { diff --git a/examples/server/tests/features/tool_call.feature b/examples/server/tests/features/tool_call.feature index 6cc3e2174..ae5326dd5 100644 --- a/examples/server/tests/features/tool_call.feature +++ b/examples/server/tests/features/tool_call.feature @@ -31,11 +31,11 @@ Feature: llama.cpp server Examples: Prompts | template_name | n_predict | tool_name | tool_arguments | tools | | meetkai-functionary-medium-v3.1 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | - | meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "I'm sorry,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + | meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "Yes, you can."} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | | meetkai-functionary-medium-v3.2 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | | meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | - | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": ". A"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | Scenario Outline: OAI Compatibility w/ tools and auto tool_choice diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 1db87c721..e560a6850 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -384,9 +384,6 @@ static json oaicompat_completion_params_parse( } llama_params["grammar_trigger_words"] = triggers; } - if (handler.updated_tools.is_null()) { - tools = handler.updated_tools; - } if (!handler.grammar.empty()) { if (llama_params.contains("grammar")) { throw std::runtime_error("Cannot use custom grammar constraints with tools."); diff --git a/src/llama-sampling.cpp b/src/llama-sampling.cpp index 8caf9f73b..26ce63e2c 100644 --- a/src/llama-sampling.cpp +++ b/src/llama-sampling.cpp @@ -1371,7 +1371,7 @@ static struct llama_sampler_i llama_sampler_grammar_i = { /* .clone = */ llama_sampler_grammar_clone, /* .free = */ llama_sampler_grammar_free, }; - + bool llama_sampler_is_grammar_empty_impl(struct llama_sampler * gsmpl) { struct llama_sampler_grammar * ctx = (struct llama_sampler_grammar *) gsmpl->ctx; return ctx->grammar == nullptr; diff --git a/src/llama.cpp b/src/llama.cpp index e7ebc4d1f..0fd4f6760 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -21312,7 +21312,7 @@ int32_t llama_chat_apply_template( struct llama_sampler * llama_sampler_init_grammar(const struct llama_model * model, const char * grammar_str, const char * grammar_root) { return llama_sampler_init_grammar_impl(model->vocab, grammar_str, grammar_root); } - + bool llama_sampler_is_grammar_empty(struct llama_sampler * gsmpl) { return llama_sampler_is_grammar_empty_impl(gsmpl); }