Update test-tokenizer-random.py
This commit is contained in:
parent
60fd27b68d
commit
db26ba5b5c
1 changed files with 3 additions and 2 deletions
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test libllama tokenizer against AutoTokenizer using brute force random words/text generation.
|
||||
# Test libllama tokenizer == AutoTokenizer.
|
||||
# Brute force random words/text generation.
|
||||
|
||||
Sample usage:
|
||||
|
||||
|
@ -584,4 +585,4 @@ if __name__ == "__main__":
|
|||
logger.info(f"TOKENIZER: '{tokenizer}'")
|
||||
vocab_file = Path(path_vocab_format % tokenizer)
|
||||
dir_tokenizer = path_tokenizers / tokenizer
|
||||
main([str(vocab_file), str(dir_tokenizer), "--verbose"])
|
||||
main([str(vocab_file), str(dir_tokenizer), "--verbose"])
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue