vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

minor: confusing hexadecimal codepoint

Browse source
This commit is contained in:
jaime-m-p 2024-06-20 19:20:37 +02:00
parent 16a7503dcc
commit 03dbcc89f6
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/test-tokenizer-1-bpe.cpp
View file

@ -125,7 +125,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize(ctx, tokens); std::string check = llama_detokenize(ctx, tokens);
if (cp != 9601 && str != check) { if (cp != 9601 && str != check) {
fprintf(stderr, "error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
cp, check.c_str(), check.length(), str.c_str(), str.length()); cp, check.c_str(), check.length(), str.c_str(), str.length());
std::exit(3); std::exit(3);
} }

2
tests/test-tokenizer-1-spm.cpp
View file

@ -89,7 +89,7 @@ int main(int argc, char ** argv) {
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true);
std::string check = llama_detokenize(ctx, tokens); std::string check = llama_detokenize(ctx, tokens);
if (cp != 9601 && str != check) { if (cp != 9601 && str != check) {
fprintf(stderr, "error: codepoint %x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
cp, check.c_str(), check.length(), str.c_str(), str.length()); cp, check.c_str(), check.length(), str.c_str(), str.length());
std::exit(3); std::exit(3);
} }