vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Don't add bos token in test

Browse source
This commit is contained in:
goerch 2023-10-10 07:41:49 +02:00
parent 8d0c57583c
commit d0c8d14eae
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/test-tokenizer-0-falcon.cpp
View file

@ -157,7 +157,7 @@ int main(int argc, char **argv) {
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
const std::vector<llama_token> res = llama_tokenize(ctx, text, true); const std::vector<llama_token> res = llama_tokenize(ctx, text, false);
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());