From b3d022aa1a352b39797ae7367448759fef631084 Mon Sep 17 00:00:00 2001 From: Billel Mokeddem Date: Wed, 18 Dec 2024 05:46:07 +0000 Subject: [PATCH] Add comment explaining the logic behind the if statement --- convert_hf_to_gguf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 77ab5ef4a..154902252 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -527,6 +527,7 @@ class Model: if token in added_vocab: # We need to manually encode and decode the added tokens in case special characters # used for `\n` / `\t` have been manually added in the added tokens + # To avoid unexpected issues - we make sure to encode single-char tokens if len(token) == 1: token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))