vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Simplify logic

Browse source
This commit is contained in:
goerch 2023-09-14 17:12:14 +02:00
parent 5d528ed5be
commit 01b0105890
1 changed files with 4 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

10
tests/test-tokenizer-1-llama.cpp
View file

@ -98,12 +98,10 @@ int main(int argc, char **argv) {
std::string str = codepoint_to_utf8(cp); std::string str = codepoint_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_spm(ctx, tokens); std::string check = llama_detokenize_spm(ctx, tokens);
if (str != check) { if (cp != 9601 && str != check) {
if(cp != 9601) { fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%llu) instead of >%s<(%llu)\n",
fprintf(stderr, "%s : error: codepoint %d detokenizes to >%s<(%llu) instead of >%s<(%llu)\n", __func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length()); return 3;
return 3;
}
} }
} }
} }