vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert_hf : reduce usages of UNKNOWN for InternLM2

Browse source 
This makes the changes from #8321 more consistent
with the other changes made here.
This commit is contained in:
Francis Couture-Harpin 2024-07-10 17:33:04 -04:00
parent afa6119850
commit 1caa20fc7a
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
convert_hf_to_gguf.py
View file

@ -2189,7 +2189,7 @@ class InternLM2Model(Model):
toktype = SentencePieceTokenTypes.BYTE toktype = SentencePieceTokenTypes.BYTE
# take care of ununsed raw token # take care of ununsed raw token
if piece.startswith('[UNUSED'): if piece.startswith('[UNUSED'):
toktype = SentencePieceTokenTypes.UNKNOWN toktype = SentencePieceTokenTypes.UNUSED
tokens.append(text) tokens.append(text)
scores.append(score) scores.append(score)
@ -2219,7 +2219,7 @@ class InternLM2Model(Model):
if token == chat_eos_token: if token == chat_eos_token:
chat_eos_token_id = token_id chat_eos_token_id = token_id
token = token.encode("utf-8") token = token.encode("utf-8")
if toktypes[token_id] != SentencePieceTokenTypes.UNKNOWN: if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
assert(tokens[token_id] == token) assert(tokens[token_id] == token)
tokens[token_id] = token tokens[token_id] = token
scores[token_id] = -1000.0 scores[token_id] = -1000.0
@ -2238,7 +2238,7 @@ class InternLM2Model(Model):
if token == chat_eos_token: if token == chat_eos_token:
chat_eos_token_id = token_id chat_eos_token_id = token_id
token = token.encode("utf-8") token = token.encode("utf-8")
if toktypes[token_id] != SentencePieceTokenTypes.UNKNOWN: if toktypes[token_id] != SentencePieceTokenTypes.UNUSED:
assert(tokens[token_id] == token) assert(tokens[token_id] == token)
tokens[token_id] = token tokens[token_id] = token
scores[token_id] = -1000.0 scores[token_id] = -1000.0