From 5afa3eff3ac35a6d9467e349eee7e5fc287ffd08 Mon Sep 17 00:00:00 2001
From: Molly Sophia <mollysophia379@gmail.com>
Date: Mon, 12 Aug 2024 14:16:02 +0800
Subject: [PATCH] Update convert_hf_to_gguf.py

Co-authored-by: compilade <git@compilade.net>
---
 convert_hf_to_gguf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index 2f0f4d5fb..5d48980eb 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -2742,8 +2742,8 @@ class RwkvModel(Model):
                 toktypes.append(gguf.TokenType.NORMAL)
         remainder = vocab_size - len(tokens)
         assert remainder >= 0
-        for i in range(remainder):
-            tokens.append(f"<unused {i}>".encode("utf-8"))
+        for i in range(len(tokens), vocab_size):
+            tokens.append(f"[PAD{i}]".encode("utf-8"))
             toktypes.append(gguf.TokenType.UNUSED)
 
         self.gguf_writer.add_tokenizer_model("rwkv")