vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update comment

Browse source
This commit is contained in:
Galunid 2023-10-23 18:03:19 +02:00
parent a06e82bbfd
commit 1244b0060b
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/test-tokenizer-1-bpe.cpp
View file

@ -91,7 +91,7 @@ int main(int argc, char **argv) {
} }
} }
} }
// TODO: why doesn't this work for the full range of Unicodes? // Restrict to unicode planes
// for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) { // for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) {
for (uint32_t cp = 0x10000; cp < 0x00040000; ++cp) { for (uint32_t cp = 0x10000; cp < 0x00040000; ++cp) {
std::string str = codepoint_to_utf8(cp); std::string str = codepoint_to_utf8(cp);