From aa0551a504c872c6e5e870c46d89bb5867e19874 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 14 Aug 2023 20:14:55 +0300 Subject: [PATCH] tests : fix build + warnings (test-tokenizer-1 still fails) --- tests/test-tokenizer-0.cpp | 30 +++++++++++++++--------------- tests/test-tokenizer-1.cpp | 19 ++++++++++--------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index a523c320c..f973271a3 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -8,14 +8,13 @@ static std::string unescape_whitespace(llama_context* ctx, const std::vector& tokens) { std::string result; - for (int i = 0; i < tokens.size(); ++i) { + for (size_t i = 0; i < tokens.size(); ++i) { result += llama_token_to_str(ctx, tokens[i]); } return result; } -static const std::map> & k_tests() -{ +static const std::map> & k_tests() { static std::map> _k_tests = { { " ", {1, 259, }, }, { "\t", { 1, 29871, 12, }, }, @@ -29,17 +28,18 @@ static const std::map> & k_tests() { " this is πŸ¦™.cpp", { 1, 29871, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, { "w048 7tuijk dsdfhu", { 1, 281, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, { "Π½Π΅Ρ‰ΠΎ Π½Π° Π‘ΡŠΠ»Π³Π°Ρ€ΡΠΊΠΈ", { 1, 1538, 4851, 665, 1386, 29713, 1305, }, }, - { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰", { 1, 29871, 31849, 31324, 31934, 228, 162, 142, 228, 161, - 146, 228, 162, 133, 228, 161, 153, 228, 161, 186, - 31708, 228, 162, 132, 31708, 228, 161, 165, 31324, 228, - 161, 136, 228, 161, 132, 228, 161, 158, 228, 161, - 136, 228, 162, 132, 228, 161, 140, }, }, + { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰", { 1, 29871, 31849, 31324, 31934, 228, 162, 142, 228, 161, + 146, 228, 162, 133, 228, 161, 153, 228, 161, 186, + 31708, 228, 162, 132, 31708, 228, 161, 165, 31324, 228, + 161, 136, 228, 161, 132, 228, 161, 158, 228, 161, + 136, 228, 162, 132, 228, 161, 140, }, }, { "πŸš€ (normal) πŸ˜Άβ€πŸŒ«οΈ (multiple emojis concatenated) βœ… (only emoji that has its own token)", - { 1, 29871, 243, 162, 157, 131, 313, 8945, 29897, 29871, - 243, 162, 155, 185, 30722, 243, 162, 143, 174, 30598, - 313, 20787, 953, 3848, 275, 16125, 630, 29897, 29871, 31681, - 313, 6194, 953, 29877, 2397, 393, 756, 967, 1914, 5993, 29897, }, }, - }; + { 1, 29871, 243, 162, 157, 131, 313, 8945, 29897, 29871, + 243, 162, 155, 185, 30722, 243, 162, 143, 174, 30598, + 313, 20787, 953, 3848, 275, 16125, 630, 29897, 29871, 31681, + 313, 6194, 953, 29877, 2397, 393, 756, 967, 1914, 5993, 29897, }, }, + }; + return _k_tests; }; @@ -90,8 +90,8 @@ int main(int argc, char **argv) { } for (const auto & test_kv : k_tests()) { - std::vector res = llama_tokenize(ctx, test_kv.first.c_str(), true); - fprintf(stderr, "%s : '%s' tokenized to '%s'\n", + std::vector res = llama_tokenize(ctx, test_kv.first, true); + fprintf(stderr, "%s : '%s' tokenized to '%s'\n", __func__, test_kv.first.c_str(), unescape_whitespace(ctx, res).c_str()); bool correct = res.size() == test_kv.second.size(); diff --git a/tests/test-tokenizer-1.cpp b/tests/test-tokenizer-1.cpp index 122e51684..6cd96a18b 100644 --- a/tests/test-tokenizer-1.cpp +++ b/tests/test-tokenizer-1.cpp @@ -8,8 +8,9 @@ #include #include #include +#include -static std::string vocab_type(llama_context* ctx) { +static std::string vocab_type(llama_context * ctx) { return llama_n_vocab(ctx) == 32000 ? "spm": "bpe"; } @@ -32,9 +33,9 @@ static std::string escape_whitespace(const std::string& text) { return result; } -static std::string unescape_whitespace(llama_context* ctx, const std::vector& tokens) { +static std::string unescape_whitespace(llama_context * ctx, const std::vector & tokens) { std::string result; - for (int i = 0; i < tokens.size(); ++i) { + for (size_t i = 0; i < tokens.size(); ++i) { result += llama_token_to_str(ctx, tokens[i]); } return result; @@ -85,17 +86,17 @@ int main(int argc, char **argv) { if (tokens.size() == 1) { if (i != tokens[0]) { std::string backward = llama_token_to_str(ctx, tokens[0]); - fprintf(stderr, "%s : error: token %d is string %s but bpe returns token %d %s\n", + fprintf(stderr, "%s : error: token %d is string %s but bpe returns token %d %s\n", __func__, i, llama_token_to_str(ctx, i).c_str(), tokens[0], backward.c_str()); return 2; } } else { - if ((vocab_type(ctx) == "spm" && i <= 258) || + if ((vocab_type(ctx) == "spm" && i <= 258) || (vocab_type(ctx) == "bpe" && (i == 0 || i >= 100000))) { - fprintf(stderr, "%s : info: token %d is string %s and bpe returns tokens %s\n", + fprintf(stderr, "%s : info: token %d is string %s and bpe returns tokens %s\n", __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str()); } else { - fprintf(stderr, "%s : error: token %d is string %s but bpe returns tokens %s\n", + fprintf(stderr, "%s : error: token %d is string %s but bpe returns tokens %s\n", __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str()); return 2; } @@ -106,9 +107,9 @@ int main(int argc, char **argv) { for (wchar_t ch = 0x0000; ch < 0xffff; ++ch) { std::wstring wstr(1, ch); std::string str = converter.to_bytes(wstr); - std::vector tokens = llama_tokenize(ctx, escape_whitespace(str).c_str(), false); + std::vector tokens = llama_tokenize(ctx, escape_whitespace(str), false); if (tokens.size() == 1) { - fprintf(stderr, "%s : info: %s tokenized to %d \n", + fprintf(stderr, "%s : info: %s tokenized to %d \n", __func__, str.c_str(), tokens[0]); } }