diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 40c0c6b4e..36a73c9d7 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -16,9 +16,10 @@ static std::string unescape_whitespace(llama_context* ctx, const llama_token* to static const std::map> & k_tests() { static std::map> _k_tests = { - {" ", {1, 259,},}, + { " ", {1, 259, }, }, { "\t", { 1, 29871, 12, }, }, { "\n", { 1, 29871, 13, }, }, + { "\t\n", { 1, 29871, 12, 13, }, }, { "Hello world", { 1, 15043, 3186, }, }, { " Hello world", { 1, 29871, 15043, 3186, }, }, { "Hello World", { 1, 15043, 2787, }, },