diff --git a/tests/test-tokenizer-1-bpe.cpp b/tests/test-tokenizer-1-bpe.cpp
index 9c748a9fb..1f3c6c750 100644
--- a/tests/test-tokenizer-1-bpe.cpp
+++ b/tests/test-tokenizer-1-bpe.cpp
@@ -110,16 +110,16 @@ int main(int argc, char **argv) {
 
         for (int i = 0; i < nthread; ++i) {
             threads[i] = std::thread([i, nthread, ctx]() {
-                for (uint32_t cp = i; cp < 0x0010ffff; cp += nthread) {
-                    if (!( // NOLINT
-                                (cp < 0x03       || cp >  0x05)   && cp != 0x0b && cp != 0x11 &&
-                                (cp < 0x13       || cp >  0x17)   && cp != 0x19 &&
-                                (cp < 0x1c       || cp >  0x1e)   &&
-                                (cp < 0xd800     || cp >  0xdfff) &&
-                                (cp < 0x00040000 || cp >= 0x000e0000)
-                        )) {
-                        continue;
-                    }
+                for (uint32_t cp = i; cp < 0x00110000; cp += nthread) {
+                    //if (!( // NOLINT
+                    //            (cp < 0x03       || cp >  0x05)   && cp != 0x0b && cp != 0x11 &&
+                    //            (cp < 0x13       || cp >  0x17)   && cp != 0x19 &&
+                    //            (cp < 0x1c       || cp >  0x1e)   &&
+                    //            (cp < 0xd800     || cp >  0xdfff) &&
+                    //            (cp < 0x00040000 || cp >= 0x000e0000)
+                    //    )) {
+                    //    continue;
+                    //}
 
                     std::string str = unicode_cpt_to_utf8(cp);
                     std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
diff --git a/tests/test-tokenizer-1-spm.cpp b/tests/test-tokenizer-1-spm.cpp
index cb37b4880..9f3d8d148 100644
--- a/tests/test-tokenizer-1-spm.cpp
+++ b/tests/test-tokenizer-1-spm.cpp
@@ -80,10 +80,10 @@ int main(int argc, char ** argv) {
 
         for (int i = 0; i < nthread; ++i) {
             threads[i] = std::thread([i, nthread, ctx]() {
-                for (uint32_t cp = i; cp < 0x0010ffff; cp += nthread) {
-                    if (cp >= 0xd800 && cp <= 0xdfff) {
-                        continue;
-                    }
+                for (uint32_t cp = i; cp < 0x00110000; cp += nthread) {
+                    //if (cp >= 0xd800 && cp <= 0xdfff) {
+                    //    continue;
+                    //}
 
                     std::string str = unicode_cpt_to_utf8(cp);
                     std::vector<llama_token> tokens = llama_tokenize(ctx, str, false, true);