From d8d2f370dc97f3bba3ecc1f9fc6b6853a0794028 Mon Sep 17 00:00:00 2001 From: Billel Mokeddem Date: Wed, 18 Dec 2024 07:23:35 +0000 Subject: [PATCH] Add a log message to better track the when the following line of code is triggered --- convert_hf_to_gguf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 154902252..cd5dd9435 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -529,6 +529,7 @@ class Model: # used for `\n` / `\t` have been manually added in the added tokens # To avoid unexpected issues - we make sure to encode single-char tokens if len(token) == 1: + logger.info("Ecode-Decode special characters using AutoTokenizer") token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False)) if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token):