From 0b6ffa580c5b6db743d21cab82a048dcfc496d6b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 16 Dec 2023 16:05:35 +0200 Subject: [PATCH] convert : revert "added_tokens_decoder" change --- convert-hf-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 8aaf67384..e71a96c48 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -249,7 +249,7 @@ class Model: toktypes.append(gguf.TokenType.USER_DEFINED) elif reverse_vocab[i] in added_vocab: tokens.append(reverse_vocab[i]) - if hasattr(tokenizer, "added_tokens_decoder") and tokenizer.added_tokens_decoder[i].special: + if tokenizer.added_tokens_decoder[i].special: toktypes.append(gguf.TokenType.CONTROL) else: toktypes.append(gguf.TokenType.USER_DEFINED)