From f428652b2ab852fe84f39d6877a8f38d63fa4df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20de=20Vries?= Date: Wed, 14 Feb 2024 14:05:57 +0100 Subject: [PATCH] fix(gguf-py): special tokens are no longer skipped when add__token is set to false --- gguf-py/gguf/vocab.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index cd1942975..1db08b598 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -152,10 +152,6 @@ class SpecialVocab: add_entry = tokenizer_config.get(f'add_{typ}_token') if isinstance(add_entry, bool): self.add_special_token[typ] = add_entry - if not added_tokens: - # We will need this to get the content for the token, so if it's empty - # may as well just give up. - continue entry = tokenizer_config.get(f'{typ}_token') if isinstance(entry, str): tc_content = entry