Add a log message to better track the when the following line of code is triggered

This commit is contained in:
Billel Mokeddem 2024-12-18 07:23:35 +00:00
parent b3d022aa1a
commit d8d2f370dc

View file

@ -529,6 +529,7 @@ class Model:
# used for `\n` / `\t` have been manually added in the added tokens
# To avoid unexpected issues - we make sure to encode single-char tokens
if len(token) == 1:
logger.info("Ecode-Decode special characters using AutoTokenizer")
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))
if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token):