vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Restrict bpe tokenizer tests to unicode planes

Browse source
This commit is contained in:
Galunid 2023-10-23 17:52:35 +02:00
parent 226f0a857d
commit a06e82bbfd
1 changed files with 11 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

13
tests/test-tokenizer-1-bpe.cpp
View file

@ -93,7 +93,17 @@ int main(int argc, char **argv) {
} }
// TODO: why doesn't this work for the full range of Unicodes? // TODO: why doesn't this work for the full range of Unicodes?
// for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) { // for (uint32_t cp = 0x10000; cp < 0x0010ffff; ++cp) {
for (uint32_t cp = 0x10000; cp < 0x00080000; ++cp) { for (uint32_t cp = 0x10000; cp < 0x00040000; ++cp) {
std::string str = codepoint_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_bpe(ctx, tokens);
if (str != check) {
fprintf(stderr, "%s : error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
__func__, cp, check.c_str(), check.length(), str.c_str(), str.length());
return 4;
}
}
for (uint32_t cp = 0x000e0000; cp < 0x0010ffff; ++cp) {
std::string str = codepoint_to_utf8(cp); std::string str = codepoint_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_bpe(ctx, tokens); std::string check = llama_detokenize_bpe(ctx, tokens);
@ -103,7 +113,6 @@ int main(int argc, char **argv) {
return 4; return 4;
} }
} }
llama_free_model(model); llama_free_model(model);
llama_free(ctx); llama_free(ctx);