vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

py : pad with unknown tokens when data is missing

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-01-16 14:03:57 +02:00
parent 9b464b4e81
commit a1372737e0
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 9 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

9
convert.py
View file

@ -1098,6 +1098,15 @@ class OutputFile:
scores.append(score)
toktypes.append(toktype)
# pad with unknown tokens and print warnings
# ref: https://github.com/ggerganov/llama.cpp/issues/4958
if len(tokens) < vocab.vocab_size:
for i in range(len(tokens), vocab.vocab_size):
tokens.append(f"<unk{i}>".encode("utf-8"))
scores.append(-1000.0)
toktypes.append(gguf.TokenType.UNKNOWN)
print(f"Warning: token {i} not found in vocab - padding with {tokens[-1]}")
return tokens, scores, toktypes
def add_meta_vocab(self, vocab: Vocab) -> None: