vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add simple test for special tokens

Browse source
This commit is contained in:
Igor Pissolati 2023-08-07 17:31:13 -03:00
parent d9791bb48b
commit 6f7dabab44
1 changed files with 6 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tests/test-tokenizer-0.cpp
View file

@ -14,6 +14,8 @@ static const std::map<std::string, std::vector<llama_token>> & k_tests()
{ " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, },
{ "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, },
{ "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, },
{ "<🦙>test extra_id_1 test", { 1, 32003, 1688, 29871, 32001, 259, 1243, }, },
{ "<🦙>test extra_id_100 test", { 1, 32003, 1688, 29871, 32002, 1243, }, },
}; };
return _k_tests; return _k_tests;
}; };
@ -46,6 +48,10 @@ int main(int argc, char **argv) {
return 1; return 1;
} }
llama_add_special_token(model, "extra_id_1", 32001);
llama_add_special_token(model, "extra_id_100", 32002);
llama_add_special_token(model, "<🦙>", 32003);
ctx = llama_new_context_with_model(model, lparams); ctx = llama_new_context_with_model(model, lparams);
if (ctx == NULL) { if (ctx == NULL) {