vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert.py: Set gpt2 as tokenizer model when using BPE

Browse source
This commit is contained in:
KerfuffleV2 2023-08-29 12:01:59 -06:00
parent 58fa4dc870
commit ce005285aa
1 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

7
convert.py
View file

@ -846,7 +846,12 @@ class OutputFile:
scores.append(score)
toktypes.append(toktype)
self.gguf.add_tokenizer_model("llama")
if isinstance(vocab, SentencePieceVocab):
self.gguf.add_tokenizer_model("llama")
elif isinstance(vocab, BpeVocab):
self.gguf.add_tokenizer_model("gpt2")
else:
raise ValueError(f'Unknown vocab type: Not BpeVocab or SentencePieceVocab')
self.gguf.add_token_list(tokens)
self.gguf.add_token_scores(scores)
self.gguf.add_token_types(toktypes)