vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix lazy trigger handling

Browse source
This commit is contained in:
Olivier Chafik 2025-01-22 19:08:19 +00:00
parent c2d836f9d0
commit 46415d7a51
2 changed files with 7 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

3
examples/server/server.cpp
View file

@ -402,8 +402,7 @@ struct server_task {
{ {
const auto grammar_trigger_words = data.find("grammar_trigger_words"); const auto grammar_trigger_words = data.find("grammar_trigger_words");
if (grammar_trigger_words != data.end()) { if (grammar_trigger_words != data.end()) {
auto words = to_string_vec(*grammar_trigger_words); for (const auto & word : to_string_vec(*grammar_trigger_words)) {
for (const auto & word : params.sampling.grammar_trigger_words) {
auto ids = common_tokenize(vocab, word, /* add_special= */ false, /* parse_special= */ true); auto ids = common_tokenize(vocab, word, /* add_special= */ false, /* parse_special= */ true);
if (ids.size() == 1) { if (ids.size() == 1) {
params.sampling.grammar_trigger_tokens.push_back(ids[0]); params.sampling.grammar_trigger_tokens.push_back(ids[0]);

12
examples/server/tests/unit/test_chat_completion.py
View file

@ -273,12 +273,12 @@ def test_completion_with_required_tool(template_name: str, n_predict: int, tool:
@pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [ @pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [
("meetkai-functionary-medium-v3.1", 32, [], None), ("meetkai-functionary-medium-v3.1", 128, [], None),
("meetkai-functionary-medium-v3.1", 32, [TEST_TOOL], None), ("meetkai-functionary-medium-v3.1", 128, [TEST_TOOL], None),
("meetkai-functionary-medium-v3.1", 32, [PYTHON_TOOL], 'none'), ("meetkai-functionary-medium-v3.1", 128, [PYTHON_TOOL], 'none'),
("meetkai-functionary-medium-v3.2", 32, [], None), ("meetkai-functionary-medium-v3.2", 128, [], None),
("meetkai-functionary-medium-v3.2", 32, [TEST_TOOL], None), ("meetkai-functionary-medium-v3.2", 128, [TEST_TOOL], None),
("meetkai-functionary-medium-v3.2", 32, [PYTHON_TOOL], 'none'), ("meetkai-functionary-medium-v3.2", 128, [PYTHON_TOOL], 'none'),
("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [], None), ("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [], None),
("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [TEST_TOOL], None), ("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [TEST_TOOL], None),
("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [PYTHON_TOOL], 'none'), ("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [PYTHON_TOOL], 'none'),