From af45703f7458069cfad72f403c88e8d27f672613 Mon Sep 17 00:00:00 2001
From: jaime-m-p <>
Date: Thu, 23 May 2024 20:16:34 +0200
Subject: [PATCH] Add WPM models for testing

---
 tests/test-tokenizer-random.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py
index b3ec4d0da..ec1b2837c 100644
--- a/tests/test-tokenizer-random.py
+++ b/tests/test-tokenizer-random.py
@@ -328,8 +328,10 @@ if __name__ == "__main__":
     # import os
     # tokenizers = os.listdir(path_tokenizers)
     tokenizers = [
-        "llama-spm",   # SPM
-        "phi-3",       # SPM
+        # "llama-spm",   # SPM
+        # "phi-3",       # SPM
+        "jina-v2-en",  # WPM
+        "bert-bge",    # WPM
     ]
 
     for tokenizer in tokenizers: