vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tool-call: fix passing of tools to template + allow agent to finish

Browse source
This commit is contained in:
ochafik 2024-09-27 06:24:22 +01:00
parent f62e688387
commit e33b342da7
8 changed files with 8 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

1
common/tool-call.h
View file

@ -21,7 +21,6 @@ struct llama_tool_call_handler {
std::string grammar;
std::vector<std::string> grammar_trigger_words;
std::vector<std::string> additional_stop_words;
nlohmann::ordered_json updated_tools;
};
llama_tool_calls parse_tool_calls(llama_tool_call_style style, const nlohmann::ordered_json & tools, const std::string& input);

3
examples/agent/README.md
View file

@ -15,8 +15,7 @@
- Run some tools inside a docker container (check http://localhost:8088/docs once running):
```bash
docker run -p 8088:8088 -w /src \
-v $PWD/examples/agent:/src \
docker run -p 8088:8088 -w /src -v $PWD/examples/agent:/src \
--rm -it ghcr.io/astral-sh/uv:python3.12-alpine \
uv run fastify.py --port 8088 tools.py
```

1
examples/agent/run.py
View file

@ -182,6 +182,7 @@ def main(
else:
assert content
print(content)
return
i += 1

3
examples/server/server.cpp
View file

@ -986,6 +986,7 @@ struct server_context {
{
slot.antiprompts.clear();
slot.sparams.grammar_trigger_words.clear();
auto copy_string_array = [&](const json & data, const std::string & key, std::vector<std::string> & vec) {
const auto & arr = data.find(key);
@ -1004,7 +1005,7 @@ struct server_context {
copy_string_array(data, "grammar_trigger_words", slot.sparams.grammar_trigger_words);
slot.antiprompts.build(ctx, stop_words, slot.sparams.grammar_trigger_words);
}
{

4
examples/server/tests/features/tool_call.feature
View file

@ -31,11 +31,11 @@ Feature: llama.cpp server
Examples: Prompts
| template_name | n_predict | tool_name | tool_arguments | tools |
| meetkai-functionary-medium-v3.1 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "I'm sorry,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "Yes, you can."} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meetkai-functionary-medium-v3.2 | 128 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": ". A"} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
Scenario Outline: OAI Compatibility w/ tools and auto tool_choice

3
examples/server/utils.hpp
View file

@ -384,9 +384,6 @@ static json oaicompat_completion_params_parse(
}
llama_params["grammar_trigger_words"] = triggers;
}
if (handler.updated_tools.is_null()) {
tools = handler.updated_tools;
}
if (!handler.grammar.empty()) {
if (llama_params.contains("grammar")) {
throw std::runtime_error("Cannot use custom grammar constraints with tools.");

2
src/llama-sampling.cpp
View file

@ -1371,7 +1371,7 @@ static struct llama_sampler_i llama_sampler_grammar_i = {
/* .clone = */ llama_sampler_grammar_clone,
/* .free = */ llama_sampler_grammar_free,
};
bool llama_sampler_is_grammar_empty_impl(struct llama_sampler * gsmpl) {
struct llama_sampler_grammar * ctx = (struct llama_sampler_grammar *) gsmpl->ctx;
return ctx->grammar == nullptr;

2
src/llama.cpp
View file

@ -21312,7 +21312,7 @@ int32_t llama_chat_apply_template(
struct llama_sampler * llama_sampler_init_grammar(const struct llama_model * model, const char * grammar_str, const char * grammar_root) {
return llama_sampler_init_grammar_impl(model->vocab, grammar_str, grammar_root);
}
bool llama_sampler_is_grammar_empty(struct llama_sampler * gsmpl) {
return llama_sampler_is_grammar_empty_impl(gsmpl);
}