Update test-tokenizer-random.py
This commit is contained in:
parent
3275e29360
commit
5edd022d6a
1 changed files with 2 additions and 2 deletions
|
@ -3,13 +3,13 @@
|
|||
# Test libllama tokenizer == AutoTokenizer.
|
||||
# Brute force random words/text generation.
|
||||
#
|
||||
#Sample usage:
|
||||
# Sample usage:
|
||||
#
|
||||
# python3 tests/test-tokenizer-random.py ./models/ggml-vocab-llama-bpe.gguf ./models/tokenizers/llama-bpe
|
||||
#
|
||||
|
||||
from __future__ import annotations
|
||||
#
|
||||
|
||||
import time
|
||||
import logging
|
||||
import argparse
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue