vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tets: skip unicode surrogaes and undefined

Browse source
This commit is contained in:
jaime-m-p 2024-06-23 20:56:32 +02:00
parent 0cf2989b6c
commit 38d54b3c39
2 changed files with 8 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

13
tests/test-tokenizer-1-bpe.cpp
View file

@ -117,15 +117,10 @@ int main(int argc, char **argv) {
for (int i = 0; i < nthread; ++i) { for (int i = 0; i < nthread; ++i) {
threads[i] = std::thread([i, nthread, ctx, &errcode]() { threads[i] = std::thread([i, nthread, ctx, &errcode]() {
for (uint32_t cp = i; !errcode && cp < 0x00110000; cp += nthread) { for (uint32_t cp = i; !errcode && cp < 0x00110000; cp += nthread) {
//if (!( // NOLINT if ((0x0000D800 <= cp && cp <= 0x0000DFFF) || // surrogates \p{Cs}
// (cp < 0x03 || cp > 0x05) && cp != 0x0b && cp != 0x11 && (0x00040000 <= cp && cp <= 0x000E0000)) { // undefined \p{Cn}
// (cp < 0x13 || cp > 0x17) && cp != 0x19 && continue;
// (cp < 0x1c || cp > 0x1e) && }
// (cp < 0xd800 || cp > 0xdfff) &&
// (cp < 0x00040000 || cp >= 0x000e0000)
// )) {
// continue;
//}
std::string str = unicode_cpt_to_utf8(cp); std::string str = unicode_cpt_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);

7
tests/test-tokenizer-1-spm.cpp
View file

@ -87,9 +87,10 @@ int main(int argc, char ** argv) {
for (int i = 0; i < nthread; ++i) { for (int i = 0; i < nthread; ++i) {
threads[i] = std::thread([i, nthread, ctx, &errcode]() { threads[i] = std::thread([i, nthread, ctx, &errcode]() {
for (uint32_t cp = i; !errcode && cp < 0x00110000; cp += nthread) { for (uint32_t cp = i; !errcode && cp < 0x00110000; cp += nthread) {
//if (cp >= 0xd800 && cp <= 0xdfff) { if ((0x0000D800 <= cp && cp <= 0x0000DFFF) || // surrogates \p{Cs}
// continue; (0x00040000 <= cp && cp <= 0x000E0000)) { // undefined \p{Cn}
//} continue;
}
std::string str = unicode_cpt_to_utf8(cp); std::string str = unicode_cpt_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true); std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true);