Update bruteforce test: add more models

2024-07-04 22:38:12 +02:00 · 2024-07-04 22:38:12 +02:00 · 4db8c0d5d9
commit 4db8c0d5d9
parent 11ac641c1e
1 changed files with 4 additions and 0 deletions
--- a/tests/test-tokenizer-random.py
+++ b/tests/test-tokenizer-random.py
@ -526,6 +526,8 @@ if __name__ == "__main__":
    tokenizers = [
        "llama-spm",      # SPM
        "phi-3",          # SPM
+        "gemma",          # SPM
+        "gemma-2",        # SPM
        "baichuan",       # SPM
        "bert-bge",       # WPM
        "jina-v2-en",     # WPM
@ -546,6 +548,8 @@ if __name__ == "__main__":
        "smaug-bpe",      # BPE
        "poro-chat",      # BPE
        "jina-v2-code",   # BPE
+        "viking",         # BPE
+        "jais",           # BPE
    ]

    logger.info("=" * 50)