vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Comment cosmetics

Browse source
This commit is contained in:
Galunid 2023-10-23 19:58:35 +02:00
parent 1244b0060b
commit 38de5d0458
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/test-tokenizer-1-bpe.cpp
View file

@ -91,7 +91,7 @@ int main(int argc, char **argv) {
} }
} }
} }
// Restrict to unicode planes // Restrict to assigned unicode planes
// for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) { // for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) {
for (uint32_t cp = 0x10000; cp < 0x00040000; ++cp) { for (uint32_t cp = 0x10000; cp < 0x00040000; ++cp) {
std::string str = codepoint_to_utf8(cp); std::string str = codepoint_to_utf8(cp);