vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update convert_hf_to_gguf.py

Browse source 
Co-authored-by: compilade <git@compilade.net>
This commit is contained in:
Molly Sophia 2024-08-12 14:16:02 +08:00
parent ae9936a80d
commit 5afa3eff3a
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
convert_hf_to_gguf.py
View file

@ -2742,8 +2742,8 @@ class RwkvModel(Model):
toktypes.append(gguf.TokenType.NORMAL) toktypes.append(gguf.TokenType.NORMAL)
remainder = vocab_size - len(tokens) remainder = vocab_size - len(tokens)
assert remainder >= 0 assert remainder >= 0
for i in range(remainder): for i in range(len(tokens), vocab_size):
tokens.append(f"<unused {i}>".encode("utf-8")) tokens.append(f"[PAD{i}]".encode("utf-8"))
toktypes.append(gguf.TokenType.UNUSED) toktypes.append(gguf.TokenType.UNUSED)
self.gguf_writer.add_tokenizer_model("rwkv") self.gguf_writer.add_tokenizer_model("rwkv")