From ae9936a80d6934ca9f80793c0803f39d541b15a4 Mon Sep 17 00:00:00 2001 From: Molly Sophia Date: Mon, 12 Aug 2024 14:14:56 +0800 Subject: [PATCH] Update convert_hf_to_gguf.py Co-authored-by: compilade --- convert_hf_to_gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 5eea3149f..2f0f4d5fb 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -2737,7 +2737,7 @@ class RwkvModel(Model): token = token.encode("utf-8") if isinstance(token, str) else token assert isinstance(token, bytes) assert len(token) == token_len - token_text: str = str(token)[2:-1] + token_text: str = repr(token)[2:-1] # "b'\xff'" -> "\xff" tokens.append(token_text.encode("utf-8")) toktypes.append(gguf.TokenType.NORMAL) remainder = vocab_size - len(tokens)