From 46415d7a51ea775387efb1d8f62b5f356c8e93f1 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Wed, 22 Jan 2025 19:08:19 +0000 Subject: [PATCH] Fix lazy trigger handling --- examples/server/server.cpp | 3 +-- examples/server/tests/unit/test_chat_completion.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 05ed61b9b..939e6c36a 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -402,8 +402,7 @@ struct server_task { { const auto grammar_trigger_words = data.find("grammar_trigger_words"); if (grammar_trigger_words != data.end()) { - auto words = to_string_vec(*grammar_trigger_words); - for (const auto & word : params.sampling.grammar_trigger_words) { + for (const auto & word : to_string_vec(*grammar_trigger_words)) { auto ids = common_tokenize(vocab, word, /* add_special= */ false, /* parse_special= */ true); if (ids.size() == 1) { params.sampling.grammar_trigger_tokens.push_back(ids[0]); diff --git a/examples/server/tests/unit/test_chat_completion.py b/examples/server/tests/unit/test_chat_completion.py index 75a3262c9..4bbd10c0e 100644 --- a/examples/server/tests/unit/test_chat_completion.py +++ b/examples/server/tests/unit/test_chat_completion.py @@ -273,12 +273,12 @@ def test_completion_with_required_tool(template_name: str, n_predict: int, tool: @pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [ - ("meetkai-functionary-medium-v3.1", 32, [], None), - ("meetkai-functionary-medium-v3.1", 32, [TEST_TOOL], None), - ("meetkai-functionary-medium-v3.1", 32, [PYTHON_TOOL], 'none'), - ("meetkai-functionary-medium-v3.2", 32, [], None), - ("meetkai-functionary-medium-v3.2", 32, [TEST_TOOL], None), - ("meetkai-functionary-medium-v3.2", 32, [PYTHON_TOOL], 'none'), + ("meetkai-functionary-medium-v3.1", 128, [], None), + ("meetkai-functionary-medium-v3.1", 128, [TEST_TOOL], None), + ("meetkai-functionary-medium-v3.1", 128, [PYTHON_TOOL], 'none'), + ("meetkai-functionary-medium-v3.2", 128, [], None), + ("meetkai-functionary-medium-v3.2", 128, [TEST_TOOL], None), + ("meetkai-functionary-medium-v3.2", 128, [PYTHON_TOOL], 'none'), ("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [], None), ("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [TEST_TOOL], None), ("meta-llama-Meta-Llama-3.1-8B-Instruct", 128, [PYTHON_TOOL], 'none'),