diff --git a/tests/test-tokenizer-0-bpe.py b/tests/test-tokenizer-0-bpe.py index b1cf43b50..d412ce039 100644 --- a/tests/test-tokenizer-0-bpe.py +++ b/tests/test-tokenizer-0-bpe.py @@ -1,4 +1,11 @@ # tests with BPE tokenizer +# +# sample usage: +# +# python3 tests/test-tokenizer-0-bpe.py ~/Data/huggingface/Meta-Llama-3-8B-Instruct/ +# python3 tests/test-tokenizer-0-bpe.py ~/Data/huggingface/falcon-7b/ +# python3 tests/test-tokenizer-0-bpe.py ~/Data/huggingface/deepseek-coder-6.7b-instruct/ +# import argparse diff --git a/tests/test-tokenizer-0-spm.py b/tests/test-tokenizer-0-spm.py index 6dbfd9412..f2d3b6e88 100644 --- a/tests/test-tokenizer-0-spm.py +++ b/tests/test-tokenizer-0-spm.py @@ -1,4 +1,11 @@ # tests with SPM tokenizer +# +# sample usage: +# +# python3 tests/test-tokenizer-0-spm.py ~/Data/huggingface/Llama-2-7b-hf/ +# python3 tests/test-tokenizer-0-spm.py ~/Data/huggingface/CodeLlama-34b-Instruct-hf/ +# + import argparse