vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add WPM models for testing

Browse source
This commit is contained in:
jaime-m-p 2024-05-23 20:16:34 +02:00
parent 2a38e5fa88
commit af45703f74
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tests/test-tokenizer-random.py
View file

@ -328,8 +328,10 @@ if __name__ == "__main__":
# import os # import os
# tokenizers = os.listdir(path_tokenizers) # tokenizers = os.listdir(path_tokenizers)
tokenizers = [ tokenizers = [
"llama-spm", # SPM # "llama-spm", # SPM
"phi-3", # SPM # "phi-3", # SPM
"jina-v2-en", # WPM
"bert-bge", # WPM
] ]
for tokenizer in tokenizers: for tokenizer in tokenizers: