From 54e9f23b8a17f75b2aafba6aa4044fff37776f21 Mon Sep 17 00:00:00 2001
From: jaime-m-p <>
Date: Mon, 3 Jun 2024 01:44:21 +0200
Subject: [PATCH] Fix previous commit

---
 tests/test-tokenizer-random.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py
index f699af022..52f589511 100644
--- a/tests/test-tokenizer-random.py
+++ b/tests/test-tokenizer-random.py
@@ -197,11 +197,6 @@ def generator_random_added_tokens(tokenizer, iterations=100) -> Iterator[str]:
         yield "".join(words)
 
 
-def generator_vocab_words(vocab: list[str]) -> Iterator[str]:
-    """Brute force check all vocab words"""
-    yield from vocab
-
-
 def generator_random_chars(iterations=100) -> Iterator[str]:
     """Brute force random text with simple characters"""