vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cleanup

Browse source
This commit is contained in:
goerch 2023-09-18 23:22:17 +02:00
parent 208d3d7cda
commit 407f76d9b8
1 changed files with 0 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/test-tokenizer-1-falcon.cpp
View file

@ -89,7 +89,6 @@ int main(int argc, char **argv) {
} }
} }
for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) { for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) {
// for (uint32_t cp = 0x10000; cp < 0x0080000; ++cp) {
try { try {
std::string str = codepoint_to_utf8(cp); std::string str = codepoint_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);