Test l/r-strip for more than 4 spaces
This commit is contained in:
parent
e8b3955346
commit
9307c3fd46
1 changed files with 1 additions and 1 deletions
|
@ -273,7 +273,7 @@ def generator_apostrophe() -> Iterator[str]:
|
|||
|
||||
|
||||
def generator_added_lr_strip(tokenizer: TokenizerGroundtruth) -> Iterator[str]:
|
||||
WHITESPACES = ["", " ", " ", "\n", "\r\n", "\n\n", "\t", "\t\t"]
|
||||
WHITESPACES = ["", " ", " ", "\n", "\r\n", "\n\n", "\t", "\t\t", " "]
|
||||
all_tokens = list(sorted(set(tokenizer.special_tokens + tokenizer.added_tokens)))
|
||||
for token in all_tokens:
|
||||
for lstrip in WHITESPACES:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue