vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix added_tokens crashes

Browse source
This commit is contained in:
Galunid 2023-10-22 21:21:31 +02:00
parent 01533762b8
commit e39905050f
1 changed files with 10 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

14
convert-stablelm-hf-to-gguf.py
View file

@ -112,14 +112,20 @@ vocab_size = hparams.get("vocab_size", len(tokenizer.vocab))
assert max(tokenizer.vocab.values()) < vocab_size assert max(tokenizer.vocab.values()) < vocab_size
reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()} reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
added_vocab = tokenizer.get_added_vocab()
for i in range(vocab_size): for i in range(vocab_size):
tokens.append(reverse_vocab[i] if i in reverse_vocab else f"[PAD{i}]") if i not in reverse_vocab:
scores.append(0.0) # dummy tokens.append(f"[PAD{i}]")
toktypes.append(gguf.TokenType.NORMAL) toktypes.append(gguf.TokenType.USER_DEFINED)
elif reverse_vocab[i] in added_vocab:
tokens.append(reverse_vocab[i])
toktypes.append(gguf.TokenType.USER_DEFINED)
else:
tokens.append(reverse_vocab[i])
toktypes.append(gguf.TokenType.NORMAL)
gguf_writer.add_token_list(tokens) gguf_writer.add_token_list(tokens)
gguf_writer.add_token_scores(scores)
gguf_writer.add_token_types(toktypes) gguf_writer.add_token_types(toktypes)
special_vocab = gguf.SpecialVocab(dir_model, load_merges = True) special_vocab = gguf.SpecialVocab(dir_model, load_merges = True)