convert_hf_to_gguf: rwkv tokenizer: Don't escape sequences manually
Signed-off-by: Molly Sophia <mollysophia379@gmail.com>
This commit is contained in:
parent
18decea3ed
commit
7f2e370fa2
1 changed files with 1 additions and 3 deletions
|
@ -2737,9 +2737,7 @@ class RwkvModel(Model):
|
|||
token = token.encode("utf-8") if isinstance(token, str) else token
|
||||
assert isinstance(token, bytes)
|
||||
assert len(token) == token_len
|
||||
token_text: str = ""
|
||||
for b in token:
|
||||
token_text += f"\\x{b:02x}"
|
||||
token_text: str = str(token)[2:-1]
|
||||
tokens.append(token_text.encode("utf-8"))
|
||||
toktypes.append(gguf.TokenType.NORMAL)
|
||||
remainder = vocab_size - len(tokens)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue