vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix previous commit

Browse source
This commit is contained in:
jaime-m-p 2024-06-03 01:44:21 +02:00
parent 83cac0ec83
commit 54e9f23b8a
1 changed files with 0 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

5
tests/test-tokenizer-random.py
View file

@ -197,11 +197,6 @@ def generator_random_added_tokens(tokenizer, iterations=100) -> Iterator[str]:
yield "".join(words)
def generator_vocab_words(vocab: list[str]) -> Iterator[str]:
"""Brute force check all vocab words"""
yield from vocab
def generator_random_chars(iterations=100) -> Iterator[str]:
"""Brute force random text with simple characters"""