From 4db8c0d5d978c87b30d9c27edfa3e3261c432b61 Mon Sep 17 00:00:00 2001
From: jaime-m-p <>
Date: Thu, 4 Jul 2024 22:38:12 +0200
Subject: [PATCH] Update bruteforce test: add more models

---
 tests/test-tokenizer-random.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py
index 64ee151fb..e0a4cec05 100644
--- a/tests/test-tokenizer-random.py
+++ b/tests/test-tokenizer-random.py
@@ -526,6 +526,8 @@ if __name__ == "__main__":
     tokenizers = [
         "llama-spm",      # SPM
         "phi-3",          # SPM
+        "gemma",          # SPM
+        "gemma-2",        # SPM
         "baichuan",       # SPM
         "bert-bge",       # WPM
         "jina-v2-en",     # WPM
@@ -546,6 +548,8 @@ if __name__ == "__main__":
         "smaug-bpe",      # BPE
         "poro-chat",      # BPE
         "jina-v2-code",   # BPE
+        "viking",         # BPE
+        "jais",           # BPE
     ]
 
     logger.info("=" * 50)