From ee1e10e21ea6b2f2a85b0244fc7923cdbbd2d4ae Mon Sep 17 00:00:00 2001 From: ochafik Date: Sat, 18 Jan 2025 02:52:40 +0000 Subject: [PATCH] Normalize newlines in test-chat-templates for windows tests --- tests/test-chat-template.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index 3bd11a1f0..d9e251240 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -9,6 +9,15 @@ #include "common.h" #include "chat-template.hpp" +static std::string normalize_newlines(const std::string & s) { +#ifdef _WIN32 + static const std::regex nl_regex("\r\n"); + return std::regex_replace(s, nl_regex, "\n"); +#else + return s; +#endif +} + int main(void) { std::vector conversation { {"system", "You are a helpful assistant"}, @@ -300,8 +309,8 @@ int main(void) { printf("\n\n=== %s (jinja) ===\n\n", test_case.name.c_str()); try { minja::chat_template tmpl(test_case.template_str, test_case.bos_token, test_case.eos_token); - auto output = tmpl.apply(messages, json(), add_generation_prompt); - auto expected_output = test_case.expected_output_jinja.empty() ? test_case.expected_output : test_case.expected_output_jinja; + auto output = normalize_newlines(tmpl.apply(messages, json(), add_generation_prompt)); + auto expected_output = normalize_newlines(test_case.expected_output_jinja.empty() ? test_case.expected_output : test_case.expected_output_jinja); if (output != expected_output) { printf("Expected:\n%s\n", expected_output.c_str()); printf("-------------------------\n");