diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 77ab5ef4a..154902252 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -527,6 +527,7 @@ class Model: if token in added_vocab: # We need to manually encode and decode the added tokens in case special characters # used for `\n` / `\t` have been manually added in the added tokens + # To avoid unexpected issues - we make sure to encode single-char tokens if len(token) == 1: token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))