vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
Georgi Gerganov 2024-04-27 00:28:36 +03:00
parent a774d7084e
commit c160818ec0
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
3 changed files with 45 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tests/test-tokenizer-0-llama-v3.cpp
View file

@ -141,12 +141,12 @@ int main(int argc, char **argv) {
llama_detokenize_bpe(ctx, test_kv.second).c_str());
fprintf(stderr, "%s : expected tokens: ", __func__);
for (const auto & t : test_kv.second) {
fprintf(stderr, "%6d, ", t);
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
}
fprintf(stderr, "\n");
fprintf(stderr, "%s : got tokens: ", __func__);
for (const auto & t : res) {
fprintf(stderr, "%6d, ", t);
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
}
fprintf(stderr, "\n");