vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding huichen's test case

Browse source
This commit is contained in:
goerch 2023-09-16 02:16:06 +02:00
parent e41209a95f
commit afc0d0d160
2 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/test-tokenizer-0-llama.cpp
View file

@ -36,6 +36,7 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests() {
{ " Hello" , { 1678, 15043, }, }, { " Hello" , { 1678, 15043, }, },
{ " Hello" , { 268, 15043, }, }, { " Hello" , { 268, 15043, }, },
{ " Hello\n Hello" , { 268, 15043, 13, 1678, 15043, }, }, { " Hello\n Hello" , { 268, 15043, 13, 1678, 15043, }, },
{ " (" , { 29871, 313, }, },
}; };
return _k_tests; return _k_tests;

6
tests/test-tokenizer-1-llama.cpp
View file

@ -87,7 +87,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens); std::string check = llama_detokenize_spm(ctx, tokens);
if (check != str) { if (check != str) {
fprintf(stderr, "%s : error: token %d detokenizes to >%s<(%zu) but tokenization of this detokenizes to >%s<(%zu)\n", fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
__func__, i, str.c_str(), str.length(), check.c_str(), check.length()); __func__, i, str.c_str(), str.length(), check.c_str(), check.length());
return 2; return 2;
} }
@ -99,7 +99,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens); std::string check = llama_detokenize_spm(ctx, tokens);
if (cp != 9601 && str != check) { if (cp != 9601 && str != check) {
fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%zu) instead of >%s<(%zu)\n", fprintf(stderr, "%s : error: codepoint %d detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); __func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
return 3; return 3;
} }
@ -110,7 +110,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens); std::string check = llama_detokenize_spm(ctx, tokens);
if (str != check) { if (str != check) {
fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%zu) instead of >%s<(%zu)\n", fprintf(stderr, "%s : error: codepoint %d detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); __func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
return 4; return 4;
} }