vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update bruteforce test: add more models

Browse source
This commit is contained in:
jaime-m-p 2024-07-04 22:38:12 +02:00
parent 11ac641c1e
commit 4db8c0d5d9
1 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tests/test-tokenizer-random.py
View file

@ -526,6 +526,8 @@ if __name__ == "__main__":
tokenizers = [ tokenizers = [
"llama-spm", # SPM "llama-spm", # SPM
"phi-3", # SPM "phi-3", # SPM
"gemma", # SPM
"gemma-2", # SPM
"baichuan", # SPM "baichuan", # SPM
"bert-bge", # WPM "bert-bge", # WPM
"jina-v2-en", # WPM "jina-v2-en", # WPM
@ -546,6 +548,8 @@ if __name__ == "__main__":
"smaug-bpe", # BPE "smaug-bpe", # BPE
"poro-chat", # BPE "poro-chat", # BPE
"jina-v2-code", # BPE "jina-v2-code", # BPE
"viking", # BPE
"jais", # BPE
] ]
logger.info("=" * 50) logger.info("=" * 50)