Add comment explaining the logic behind the if statement
This commit is contained in:
parent
fc055407b7
commit
b3d022aa1a
1 changed files with 1 additions and 0 deletions
|
@ -527,6 +527,7 @@ class Model:
|
|||
if token in added_vocab:
|
||||
# We need to manually encode and decode the added tokens in case special characters
|
||||
# used for `\n` / `\t` have been manually added in the added tokens
|
||||
# To avoid unexpected issues - we make sure to encode single-char tokens
|
||||
if len(token) == 1:
|
||||
token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue