vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix test

Browse source
This commit is contained in:
goerch 2023-07-24 13:38:25 +02:00
parent fe7508c408
commit 8253a534eb
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/test-tokenizer-1.cpp
View file

@ -91,6 +91,7 @@ int main(int argc, char **argv) {
std::string backward = llama_token_to_str(ctx, tokens[0]); std::string backward = llama_token_to_str(ctx, tokens[0]);
fprintf(stderr, "%s : error: token %d is string %s but bpe returns token %d %s\n", fprintf(stderr, "%s : error: token %d is string %s but bpe returns token %d %s\n",
__func__, i, llama_token_to_str(ctx, i).c_str(), tokens[0], backward.c_str()); __func__, i, llama_token_to_str(ctx, i).c_str(), tokens[0], backward.c_str());
return 3;
} }
} else { } else {
if (i <= 258) { if (i <= 258) {
@ -99,6 +100,7 @@ int main(int argc, char **argv) {
} else { } else {
fprintf(stderr, "%s : error: token %d is string %s but bpe returns tokens %s\n", fprintf(stderr, "%s : error: token %d is string %s but bpe returns tokens %s\n",
__func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens.data(), n).c_str()); __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens.data(), n).c_str());
return 3;
} }
} }
} }