From db26ba5b5cf6d73e5e219e1cd597fac3659eb446 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 12 Nov 2024 22:24:03 -0800 Subject: [PATCH] Update test-tokenizer-random.py --- tests/test-tokenizer-random.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py index 98fd6e40b..752f1f902 100644 --- a/tests/test-tokenizer-random.py +++ b/tests/test-tokenizer-random.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """ -Test libllama tokenizer against AutoTokenizer using brute force random words/text generation. +# Test libllama tokenizer == AutoTokenizer. +# Brute force random words/text generation. Sample usage: @@ -584,4 +585,4 @@ if __name__ == "__main__": logger.info(f"TOKENIZER: '{tokenizer}'") vocab_file = Path(path_vocab_format % tokenizer) dir_tokenizer = path_tokenizers / tokenizer - main([str(vocab_file), str(dir_tokenizer), "--verbose"]) \ No newline at end of file + main([str(vocab_file), str(dir_tokenizer), "--verbose"])