From ff8b9941610b04a88fd67d70a1a7fb6e1aa5fb43 Mon Sep 17 00:00:00 2001 From: wonjun Jang Date: Tue, 26 Dec 2023 19:25:32 +0900 Subject: [PATCH] remove unused variable --- convert.py | 1 - 1 file changed, 1 deletion(-) diff --git a/convert.py b/convert.py index a9ccb69ec..1f0c4f2f4 100755 --- a/convert.py +++ b/convert.py @@ -371,7 +371,6 @@ class VocabLoader: self.spm = None def hf_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: - tokenizer = self.tokenizer added_tokens_ids = set(self.added_tokens_dict.values()) for i in range(self.vocab_size_base):