vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test-tokenizer-0: improve output, show how many tests failed

Browse source
This commit is contained in:
Aarni Koskela 2024-05-27 20:19:23 +03:00
parent 95f84d5ce8
commit c28c99629c
1 changed files with 22 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

37
tests/test-tokenizer-0.cpp
View file

@ -175,7 +175,7 @@ int main(int argc, char **argv) {
atexit([]() { console::cleanup(); }); atexit([]() { console::cleanup(); });
#endif #endif
bool success = true; int n_failed = 0;
const auto k_tests = [&]() -> llama_tests { const auto k_tests = [&]() -> llama_tests {
if (!fname_text.empty()) { if (!fname_text.empty()) {
@ -214,22 +214,27 @@ int main(int argc, char **argv) {
} }
if (!correct) { if (!correct) {
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); fprintf(stderr, "failed test: '%s'\n", test_kv.first.c_str());
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, auto detok = llama_detokenize_bpe(ctx, res).c_str();
llama_detokenize_bpe(ctx, res).c_str(), auto expected = llama_detokenize_bpe(ctx, test_kv.second).c_str();
llama_detokenize_bpe(ctx, test_kv.second).c_str()); fprintf(stderr, "detokenized to: '%s'\n", detok);
fprintf(stderr, "%s : expected tokens: ", __func__); if(detok != expected) {
fprintf(stderr, "but we wanted: '%s'\n", expected);
} else {
fprintf(stderr, "(which matches the expected output)\n", expected);
}
fprintf(stderr, "expected tokens: \n");
for (const auto & t : test_kv.second) { for (const auto & t : test_kv.second) {
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); fprintf(stderr, "%6d '%s'\n", t, llama_token_to_piece(ctx, t).c_str());
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n");
fprintf(stderr, "%s : got tokens: ", __func__); fprintf(stderr, "got tokens: \n");
for (const auto & t : res) { for (const auto & t : res) {
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); fprintf(stderr, "%6d '%s'\n", t, llama_token_to_piece(ctx, t).c_str());
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n====================\n");
success = false; n_failed ++;
} }
} }
@ -286,7 +291,11 @@ int main(int argc, char **argv) {
llama_backend_free(); llama_backend_free();
printf("\n"); printf("\n");
printf("Tests %s\n", success ? "passed" : "failed"); if(n_failed) {
printf("%d tests failed\n", n_failed);
return success ? 0 : 3; } else {
printf("Tests passed\n");
}
return !n_failed ? 0 : 3;
} }