vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Format test-chat.cpp

Browse source
This commit is contained in:
Olivier Chafik 2025-01-30 14:09:54 +00:00
parent 5a64af6c70
commit f223df0271
1 changed files with 340 additions and 315 deletions
Download patch file Download diff file Expand all files Collapse all files

309
tests/test-chat.cpp
View file

@ -5,15 +5,15 @@
// //
// cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null // cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null
// //
#include "chat.hpp"
#include "chat-template.hpp"
#include "llama-grammar.h"
#include "unicode.h"
#include <fstream> #include <fstream>
#include <iostream> #include <iostream>
#include <string>
#include <json.hpp> #include <json.hpp>
#include <string>
#include "chat-template.hpp"
#include "chat.hpp"
#include "llama-grammar.h"
#include "unicode.h"
using json = nlohmann::ordered_json; using json = nlohmann::ordered_json;
@ -40,8 +40,7 @@ static common_chat_msg msg_from_json(const json & message) {
return ret; return ret;
} }
template <class T> template <class T> static void assert_equals(const T & expected, const T & actual) {
static void assert_equals(const T & expected, const T & actual) {
if (expected != actual) { if (expected != actual) {
std::cerr << "Expected: " << expected << std::endl; std::cerr << "Expected: " << expected << std::endl;
std::cerr << "Actual: " << actual << std::endl; std::cerr << "Actual: " << actual << std::endl;
@ -69,7 +68,8 @@ static std::string read_file(const std::string &path) {
} }
static std::unique_ptr<llama_grammar> build_grammar(const std::string & grammar_str) { static std::unique_ptr<llama_grammar> build_grammar(const std::string & grammar_str) {
return std::unique_ptr<llama_grammar>(llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0)); return std::unique_ptr<llama_grammar>(
llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0));
} }
// TODO: extract to common helper (copied from test-grammar-integration.cpp) // TODO: extract to common helper (copied from test-grammar-integration.cpp)
@ -175,7 +175,9 @@ struct delta_data {
common_chat_format format; common_chat_format format;
}; };
static delta_data init_delta(const common_chat_template & tmpl, const std::vector<std::string> & end_tokens, const json & user_message, const json & delta_message, const json & tools, const json & tool_choice) { static delta_data init_delta(const common_chat_template & tmpl, const std::vector<std::string> & end_tokens,
const json & user_message, const json & delta_message, const json & tools,
const json & tool_choice) {
common_chat_inputs inputs; common_chat_inputs inputs;
inputs.parallel_tool_calls = true; inputs.parallel_tool_calls = true;
inputs.messages = json::array(); inputs.messages = json::array();
@ -220,7 +222,9 @@ static delta_data init_delta(const common_chat_template & tmpl, const std::vecto
gets the diff, removes any end tokens and parses the result w/ the grammar, checking that gets the diff, removes any end tokens and parses the result w/ the grammar, checking that
the parsed message is the same as the test_message the parsed message is the same as the test_message
*/ */
static void test_template(const common_chat_template & tmpl, const std::vector<std::string> & end_tokens, const json & test_message, const json & tools = {}, const std::string & expected_delta = "", bool skip_grammar_test = false, bool skip_parser_test = false) { static void test_template(const common_chat_template & tmpl, const std::vector<std::string> & end_tokens,
const json & test_message, const json & tools = {}, const std::string & expected_delta = "",
bool skip_grammar_test = false, bool skip_parser_test = false) {
common_chat_msg expected_msg = msg_from_json(test_message); common_chat_msg expected_msg = msg_from_json(test_message);
auto user_message = json{ auto user_message = json{
@ -250,7 +254,8 @@ static void test_template(const common_chat_template & tmpl, const std::vector<s
// TODO: exercice lazy grammars + triggers here, instead of skipping the test // TODO: exercice lazy grammars + triggers here, instead of skipping the test
if (!skip_grammar_test) { if (!skip_grammar_test) {
if (!match_string(data.delta, grammar.get())) { if (!match_string(data.delta, grammar.get())) {
throw std::runtime_error("Failed to match delta against grammar:\n\n" + data.delta + "\n\nGrammar: " + data.grammar); throw std::runtime_error("Failed to match delta against grammar:\n\n" + data.delta +
"\n\nGrammar: " + data.grammar);
} }
} }
} }
@ -267,10 +272,7 @@ static void test_template_output_parsers() {
{ "content", {} }, { "content", {} },
{ "tool_calls", json{ { { "tool_calls", json{ {
{ "type", "function" }, { "type", "function" },
{"function", { { "function", { { "name", "special_function" }, { "arguments", "{\"arg1\": 1}" } } },
{"name", "special_function"},
{"arguments", "{\"arg1\": 1}"}
}},
} } } } } }
}; };
auto tool_call_message_with_id = json::parse(tool_call_message.dump()); auto tool_call_message_with_id = json::parse(tool_call_message.dump());
@ -281,9 +283,11 @@ static void test_template_output_parsers() {
{ "content", {} }, { "content", {} },
{ "tool_calls", json{ { { "tool_calls", json{ {
{ "type", "function" }, { "type", "function" },
{"function", { { "function",
{
{ "name", "python" }, { "name", "python" },
{"arguments", { { "arguments",
{
{ "code", "print('hey')" }, { "code", "print('hey')" },
} }, } },
} }, } },
@ -294,18 +298,21 @@ static void test_template_output_parsers() {
{ "content", {} }, { "content", {} },
{ "tool_calls", json{ { { "tool_calls", json{ {
{ "type", "function" }, { "type", "function" },
{"function", { { "function",
{
{ "name", "code_interpreter" }, { "name", "code_interpreter" },
{"arguments", { { "arguments",
{
{ "code", "print('hey')" }, { "code", "print('hey')" },
} }, } },
} }, } },
} } } } } }
}; };
common_chat_inputs inputs_no_tools; common_chat_inputs inputs_no_tools;
inputs_no_tools.messages = {{{"role", "user"}, {"content", "Hey"}}}; inputs_no_tools.messages = {
{ { "role", "user" }, { "content", "Hey" } }
};
common_chat_inputs inputs_tools = inputs_no_tools; common_chat_inputs inputs_tools = inputs_no_tools;
inputs_tools.tools = json::array(); inputs_tools.tools = json::array();
@ -315,114 +322,131 @@ static void test_template_output_parsers() {
inputs_tools_builtin.tools = json::array(); inputs_tools_builtin.tools = json::array();
inputs_tools_builtin.tools.push_back(python_tool); inputs_tools_builtin.tools.push_back(python_tool);
// {
// const common_chat_template tmpl(read_file(
// "models/templates/google-gemma-2-2b-it.jinja"), "<s>", "</s>");
// std::vector<std::string> end_tokens { "<end_of_turn>" };
// assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_params_init(tmpl, inputs_no_tools).format);
// assert_equals(COMMON_CHAT_FORMAT_GENERIC, common_chat_params_init(tmpl, inputs_tools).format);
// assert_equals(COMMON_CHAT_FORMAT_GENERIC, common_chat_params_init(common_chat_template(read_file(
// "models/templates/microsoft-Phi-3.5-mini-instruct.jinja"), "<s>", "</s>"), inputs_tools).format);
// // Generic tool calls doesn't generate / parse content-only messages symmetrically.
// assert_msg_equals(msg_from_json(text_message), common_chat_parse(
// "{\n"
// " \"response\": \"Hello, world!\"\n"
// "}",
// common_chat_params_init(tmpl, inputs_tools).format));
// test_template(tmpl, end_tokens, tool_call_message_with_id, tools,
// "{\n"
// " \"tool_calls\": [\n"
// " {\n"
// " \"name\": \"special_function\",\n"
// " \"arguments\": {\n"
// " \"arg1\": 1\n"
// " },\n"
// " \"id\": \"123456789\"\n"
// " }\n"
// " ]\n"
// "}");
// }
// {
// const common_chat_template tmpl(read_file(
// "models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja"), "<s>", "</s>");
// std::vector<std::string> end_tokens { "</s>" };
// assert_equals(COMMON_CHAT_FORMAT_MISTRAL_NEMO, common_chat_params_init(tmpl, inputs_tools).format);
// test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
// test_template(tmpl, end_tokens, tool_call_message_with_id, tools,
// "[TOOL_CALLS][{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}, \"id\": \"123456789\"}]",
// /* skip_grammar_test= */ true);
// }
// {
// const common_chat_template tmpl(read_file(
// "models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja"), "<s>", "</s>");
// std::vector<std::string> end_tokens { "<|im_end|>" };
// assert_equals(COMMON_CHAT_FORMAT_HERMES_2_PRO, common_chat_params_init(tmpl, inputs_tools).format);
// assert_equals(COMMON_CHAT_FORMAT_HERMES_2_PRO, common_chat_params_init(common_chat_template(read_file(
// "models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja"), "<s>", "</s>"), inputs_tools).format);
// assert_equals(COMMON_CHAT_FORMAT_HERMES_2_PRO, common_chat_params_init(common_chat_template(read_file(
// "models/templates/Qwen-Qwen2.5-7B-Instruct.jinja"), "<s>", "</s>"), inputs_tools).format);
// test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
// test_template(tmpl, end_tokens, tool_call_message, tools,
// "<tool_call>\n"
// "{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"
// "</tool_call>");
// test_template(tmpl, end_tokens, python_tool_call_message, tools,
// "<tool_call>\n"
// "{\"name\": \"python\", \"arguments\": {\"code\": \"print('hey')\"}}\n"
// "</tool_call>");
// }
// {
// const common_chat_template tmpl(read_file(
// "models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja"), "<s>", "</s>");
// std::vector<std::string> end_tokens { "<|eom_id|>", "<|eot_id|>" };
// assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X, common_chat_params_init(tmpl, inputs_tools).format);
// assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS, common_chat_params_init(tmpl, inputs_tools_builtin).format);
// assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS, common_chat_params_init(common_chat_template(read_file(
// "models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja"), "<s>", "</s>"), inputs_tools_builtin).format);
// // test_template(tmpl, end_tokens, text_message, tools, R"(?)", /* skip_grammar_test= */ true);
// test_template(tmpl, end_tokens, code_interpreter_tool_call_message, llama_3_1_tools,
// "<|python_tag|>code_interpreter.call(code=\"print('hey')\")");
// test_template(tmpl, end_tokens, python_tool_call_message, tools,
// "<|python_tag|>python.call(code=\"print('hey')\")");
// test_template(tmpl, end_tokens, tool_call_message, tools,
// "{\"name\": \"special_function\", \"parameters\": {\"arg1\": 1}}");
// }
// {
// const common_chat_template tmpl(read_file(
// "models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja"), "<s>", "</s>");
// std::vector<std::string> end_tokens { "<|eom_id|>", "<|eot_id|>" };
// assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X, common_chat_params_init(tmpl, inputs_tools).format);
// test_template(tmpl, end_tokens, text_message, tools,
// "Hello, world!", /* skip_grammar_test= */ true);
// test_template(tmpl, end_tokens, tool_call_message, tools,
// "{\"name\": \"special_function\", \"parameters\": {\"arg1\": 1}}");
// }
// {
// const common_chat_template tmpl(read_file(
// "models/templates/meetkai-functionary-medium-v3.1.jinja"), "<s>", "</s>");
// std::vector<std::string> end_tokens { "<|eom_id|>", "<|eot_id|>" };
// assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1, common_chat_params_init(tmpl, inputs_tools).format);
// test_template(tmpl, end_tokens, text_message, tools,
// "Hello, world!", /* skip_grammar_test= */ true);
// test_template(tmpl, end_tokens, tool_call_message, tools,
// "<function=special_function>{\"arg1\": 1}</function>");
// }
{ {
const common_chat_template tmpl(read_file( const common_chat_template tmpl(read_file("models/templates/google-gemma-2-2b-it.jinja"), "<s>", "</s>");
"models/templates/meetkai-functionary-medium-v3.2.jinja"), "<s>", "</s>"); std::vector<std::string> end_tokens{ "<end_of_turn>" };
assert_equals(COMMON_CHAT_FORMAT_CONTENT_ONLY, common_chat_params_init(tmpl, inputs_no_tools).format);
assert_equals(COMMON_CHAT_FORMAT_GENERIC, common_chat_params_init(tmpl, inputs_tools).format);
assert_equals(COMMON_CHAT_FORMAT_GENERIC,
common_chat_params_init(
common_chat_template(read_file("models/templates/microsoft-Phi-3.5-mini-instruct.jinja"),
"<s>", "</s>"),
inputs_tools)
.format);
// Generic tool calls doesn't generate / parse content-only messages symmetrically.
assert_msg_equals(msg_from_json(text_message),
common_chat_parse("{\n"
" \"response\": \"Hello, world!\"\n"
"}",
common_chat_params_init(tmpl, inputs_tools).format));
test_template(tmpl, end_tokens, tool_call_message_with_id, tools,
"{\n"
" \"tool_calls\": [\n"
" {\n"
" \"name\": \"special_function\",\n"
" \"arguments\": {\n"
" \"arg1\": 1\n"
" },\n"
" \"id\": \"123456789\"\n"
" }\n"
" ]\n"
"}");
}
{
const common_chat_template tmpl(read_file("models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja"), "<s>",
"</s>");
std::vector<std::string> end_tokens{ "</s>" };
assert_equals(COMMON_CHAT_FORMAT_MISTRAL_NEMO, common_chat_params_init(tmpl, inputs_tools).format);
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
test_template(
tmpl, end_tokens, tool_call_message_with_id, tools,
"[TOOL_CALLS][{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}, \"id\": \"123456789\"}]",
/* skip_grammar_test= */ true);
}
{
const common_chat_template tmpl(
read_file("models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja"), "<s>", "</s>");
std::vector<std::string> end_tokens{ "<|im_end|>" };
assert_equals(COMMON_CHAT_FORMAT_HERMES_2_PRO, common_chat_params_init(tmpl, inputs_tools).format);
assert_equals(
COMMON_CHAT_FORMAT_HERMES_2_PRO,
common_chat_params_init(
common_chat_template(read_file("models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja"),
"<s>", "</s>"),
inputs_tools)
.format);
assert_equals(
COMMON_CHAT_FORMAT_HERMES_2_PRO,
common_chat_params_init(
common_chat_template(read_file("models/templates/Qwen-Qwen2.5-7B-Instruct.jinja"), "<s>", "</s>"),
inputs_tools)
.format);
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
test_template(tmpl, end_tokens, tool_call_message, tools,
"<tool_call>\n"
"{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}\n"
"</tool_call>");
test_template(tmpl, end_tokens, python_tool_call_message, tools,
"<tool_call>\n"
"{\"name\": \"python\", \"arguments\": {\"code\": \"print('hey')\"}}\n"
"</tool_call>");
}
{
const common_chat_template tmpl(read_file("models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja"), "<s>",
"</s>");
std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };
assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X, common_chat_params_init(tmpl, inputs_tools).format);
assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS,
common_chat_params_init(tmpl, inputs_tools_builtin).format);
assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X_WITH_BUILTIN_TOOLS,
common_chat_params_init(
common_chat_template(read_file("models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja"),
"<s>", "</s>"),
inputs_tools_builtin)
.format);
// test_template(tmpl, end_tokens, text_message, tools, R"(?)", /* skip_grammar_test= */ true);
test_template(tmpl, end_tokens, code_interpreter_tool_call_message, llama_3_1_tools,
"<|python_tag|>code_interpreter.call(code=\"print('hey')\")");
test_template(tmpl, end_tokens, python_tool_call_message, tools,
"<|python_tag|>python.call(code=\"print('hey')\")");
test_template(tmpl, end_tokens, tool_call_message, tools,
"{\"name\": \"special_function\", \"parameters\": {\"arg1\": 1}}");
}
{
const common_chat_template tmpl(read_file("models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja"), "<s>",
"</s>");
std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };
assert_equals(COMMON_CHAT_FORMAT_LLAMA_3_X, common_chat_params_init(tmpl, inputs_tools).format);
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
test_template(tmpl, end_tokens, tool_call_message, tools,
"{\"name\": \"special_function\", \"parameters\": {\"arg1\": 1}}");
}
{
const common_chat_template tmpl(read_file("models/templates/meetkai-functionary-medium-v3.1.jinja"), "<s>",
"</s>");
std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };
assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_1_LLAMA_3_1,
common_chat_params_init(tmpl, inputs_tools).format);
test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
test_template(tmpl, end_tokens, tool_call_message, tools,
"<function=special_function>{\"arg1\": 1}</function>");
}
{
const common_chat_template tmpl(read_file("models/templates/meetkai-functionary-medium-v3.2.jinja"), "<s>",
"</s>");
std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" }; std::vector<std::string> end_tokens{ "<|eom_id|>", "<|eot_id|>" };
assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2, common_chat_params_init(tmpl, inputs_no_tools).format); assert_equals(COMMON_CHAT_FORMAT_FUNCTIONARY_V3_2, common_chat_params_init(tmpl, inputs_no_tools).format);
@ -430,32 +454,31 @@ static void test_template_output_parsers() {
test_template(tmpl, end_tokens, text_message, {}, test_template(tmpl, end_tokens, text_message, {},
"all\n" "all\n"
"Hello, world!", /* skip_grammar_test= */ true); "Hello, world!",
/* skip_grammar_test= */ true);
test_template(tmpl, end_tokens, tool_call_message, tools, test_template(tmpl, end_tokens, tool_call_message, tools,
"special_function\n" "special_function\n"
"{\"arg1\": 1}"); "{\"arg1\": 1}");
} }
{ {
const common_chat_template tmpl(read_file( const common_chat_template tmpl(read_file("models/templates/fireworks-ai-llama-3-firefunction-v2.jinja"), "<s>",
"models/templates/fireworks-ai-llama-3-firefunction-v2.jinja"), "<s>", "</s>"); "</s>");
std::vector<std::string> end_tokens{ "<|eot_id|>" }; std::vector<std::string> end_tokens{ "<|eot_id|>" };
assert_equals(COMMON_CHAT_FORMAT_FIREFUNCTION_V2, common_chat_params_init(tmpl, inputs_tools).format); assert_equals(COMMON_CHAT_FORMAT_FIREFUNCTION_V2, common_chat_params_init(tmpl, inputs_tools).format);
test_template(tmpl, end_tokens, text_message, tools, test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
"Hello, world!", /* skip_grammar_test= */ true);
test_template(tmpl, end_tokens, tool_call_message, tools, test_template(tmpl, end_tokens, tool_call_message, tools,
" functools[{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}]"); " functools[{\"name\": \"special_function\", \"arguments\": {\"arg1\": 1}}]");
} }
{ {
const common_chat_template tmpl(read_file( const common_chat_template tmpl(read_file("models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja"),
"models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja"), "<s>", "</s>"); "<s>", "</s>");
std::vector<std::string> end_tokens{ "<｜end▁of▁sentence｜>" }; std::vector<std::string> end_tokens{ "<｜end▁of▁sentence｜>" };
assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1, common_chat_params_init(tmpl, inputs_tools).format); assert_equals(COMMON_CHAT_FORMAT_DEEPSEEK_R1, common_chat_params_init(tmpl, inputs_tools).format);
test_template(tmpl, end_tokens, text_message, tools, test_template(tmpl, end_tokens, text_message, tools, "Hello, world!", /* skip_grammar_test= */ true);
"Hello, world!", /* skip_grammar_test= */ true);
test_template(tmpl, end_tokens, tool_call_message, tools, test_template(tmpl, end_tokens, tool_call_message, tools,
"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n" "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>special_function\n"
"```json\n" "```json\n"
@ -468,7 +491,9 @@ int main(int argc, char **argv) {
#ifndef _WIN32 #ifndef _WIN32
if (argc > 1) { if (argc > 1) {
common_chat_inputs inputs; common_chat_inputs inputs;
inputs.messages = {{{"role", "user"}, {"content", "Hey"}}}; inputs.messages = {
{ { "role", "user" }, { "content", "Hey" } }
};
inputs.tools = json::array({ special_function_tool }); inputs.tools = json::array({ special_function_tool });
std::cout << "| Template | Format |\n"; std::cout << "| Template | Format |\n";
@ -483,10 +508,10 @@ int main(int argc, char **argv) {
common_chat_template tmpl(read_file(path), "", ""); common_chat_template tmpl(read_file(path), "", "");
auto parts = string_split(path, "/"); auto parts = string_split(path, "/");
auto name = parts[parts.size() - 1]; auto name = parts[parts.size() - 1];
std::cout << "| " << name << " | " << common_chat_format_name(common_chat_params_init(tmpl, inputs).format) << " |\n"; std::cout << "| " << name << " | " << common_chat_format_name(common_chat_params_init(tmpl, inputs).format)
<< " |\n";
} }
} } else
else
#endif #endif
{ {
test_template_output_parsers(); test_template_output_parsers();