vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert : replacing EOS token is a hack

Browse source
This commit is contained in:
Georgi Gerganov 2024-04-21 11:15:18 +03:00
parent d0a4cc8ec8
commit 7ab0939c0d
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
convert-hf-to-gguf.py
View file

@ -2202,6 +2202,8 @@ class InternLM2Model(Model):
old_eos = special_vocab.special_token_ids["eos"] old_eos = special_vocab.special_token_ids["eos"]
if "chat" in os.path.basename(self.dir_model.absolute()): if "chat" in os.path.basename(self.dir_model.absolute()):
# For the chat model, we replace the eos with '<|im_end|>'. # For the chat model, we replace the eos with '<|im_end|>'.
# TODO: this is a hack, should be fixed
# https://github.com/ggerganov/llama.cpp/pull/6745#issuecomment-2067687048
special_vocab.special_token_ids["eos"] = self._try_get_sft_eos(tokenizer) special_vocab.special_token_ids["eos"] = self._try_get_sft_eos(tokenizer)
print(f"Replace eos:{old_eos} with a special token:{special_vocab.special_token_ids['eos']} \ print(f"Replace eos:{old_eos} with a special token:{special_vocab.special_token_ids['eos']} \
in chat mode so that the conversation can end normally.") in chat mode so that the conversation can end normally.")