vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Normalize newlines in test-chat-templates for windows tests

Browse source
This commit is contained in:
ochafik 2025-01-18 02:52:40 +00:00
parent d5fa351a24
commit ee1e10e21e
1 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

13
tests/test-chat-template.cpp
View file

@ -9,6 +9,15 @@
#include "common.h" #include "common.h"
#include "chat-template.hpp" #include "chat-template.hpp"
static std::string normalize_newlines(const std::string & s) {
#ifdef _WIN32
static const std::regex nl_regex("\r\n");
return std::regex_replace(s, nl_regex, "\n");
#else
return s;
#endif
}
int main(void) { int main(void) {
std::vector<llama_chat_message> conversation { std::vector<llama_chat_message> conversation {
{"system", "You are a helpful assistant"}, {"system", "You are a helpful assistant"},
@ -300,8 +309,8 @@ int main(void) {
printf("\n\n=== %s (jinja) ===\n\n", test_case.name.c_str()); printf("\n\n=== %s (jinja) ===\n\n", test_case.name.c_str());
try { try {
minja::chat_template tmpl(test_case.template_str, test_case.bos_token, test_case.eos_token); minja::chat_template tmpl(test_case.template_str, test_case.bos_token, test_case.eos_token);
auto output = tmpl.apply(messages, json(), add_generation_prompt); auto output = normalize_newlines(tmpl.apply(messages, json(), add_generation_prompt));
auto expected_output = test_case.expected_output_jinja.empty() ? test_case.expected_output : test_case.expected_output_jinja; auto expected_output = normalize_newlines(test_case.expected_output_jinja.empty() ? test_case.expected_output : test_case.expected_output_jinja);
if (output != expected_output) { if (output != expected_output) {
printf("Expected:\n%s\n", expected_output.c_str()); printf("Expected:\n%s\n", expected_output.c_str());
printf("-------------------------\n"); printf("-------------------------\n");