From 1029ff9028941b448143d99013dc9b60cea8f785 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Thu, 30 Jan 2025 15:13:26 +0000 Subject: [PATCH] force printing on hermes 2 model if/as it's a special token --- common/chat.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/chat.cpp b/common/chat.cpp index 2b17374d5..00ea5aee3 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -691,6 +691,8 @@ static common_chat_params common_chat_params_init_hermes_2_pro(const common_chat auto tool_call = "\"\" space " + builder.add_rule("tool_call", string_join(tool_rules, " | ")) + " \"\" space"; builder.add_rule("root", inputs.parallel_tool_calls ? "(" + tool_call + ")+" : tool_call); data.grammar_triggers.push_back({"", /* .at_start = */ false}); + // Not really a trigger but need to print this special token to get a successful parse. + data.grammar_triggers.push_back({"", /* .at_start = */ false}); }, grammar_options); data.prompt = tmpl.apply(inputs.messages, inputs.tools.empty() ? json() : inputs.tools, inputs.add_generation_prompt);