vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

One more test case...

Browse source
This commit is contained in:
goerch 2023-07-23 18:46:29 +02:00
parent e6b1a5003e
commit dba8369a39
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
tests/test-tokenizer-0.cpp
View file

@ -16,9 +16,10 @@ static std::string unescape_whitespace(llama_context* ctx, const llama_token* to
static const std::map<std::string, std::vector<llama_token>> & k_tests() static const std::map<std::string, std::vector<llama_token>> & k_tests()
{ {
static std::map<std::string, std::vector<llama_token>> _k_tests = { static std::map<std::string, std::vector<llama_token>> _k_tests = {
{" ", {1, 259,},}, { " ", {1, 259, }, },
{ "\t", { 1, 29871, 12, }, }, { "\t", { 1, 29871, 12, }, },
{ "\n", { 1, 29871, 13, }, }, { "\n", { 1, 29871, 13, }, },
{ "\t\n", { 1, 29871, 12, 13, }, },
{ "Hello world", { 1, 15043, 3186, }, }, { "Hello world", { 1, 15043, 3186, }, },
{ " Hello world", { 1, 29871, 15043, 3186, }, }, { " Hello world", { 1, 29871, 15043, 3186, }, },
{ "Hello World", { 1, 15043, 2787, }, }, { "Hello World", { 1, 15043, 2787, }, },