vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

One more test case...

Browse source
This commit is contained in:
goerch 2023-07-23 18:46:29 +02:00
parent e6b1a5003e
commit dba8369a39
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/test-tokenizer-0.cpp
View file

@ -19,6 +19,7 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests()
{ " ", {1, 259, }, }, { " ", {1, 259, }, },
{ "\t", { 1, 29871, 12, }, }, { "\t", { 1, 29871, 12, }, },
{ "\n", { 1, 29871, 13, }, }, { "\n", { 1, 29871, 13, }, },
{ "\t\n", { 1, 29871, 12, 13, }, },
{ "Hello world", { 1, 15043, 3186, }, }, { "Hello world", { 1, 15043, 3186, }, },
{ " Hello world", { 1, 29871, 15043, 3186, }, }, { " Hello world", { 1, 29871, 15043, 3186, }, },
{ "Hello World", { 1, 15043, 2787, }, }, { "Hello World", { 1, 15043, 2787, }, },