vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

change add_meta_vocab fucntion

Browse source
This commit is contained in:
wonjun Jang 2023-10-15 09:43:22 +00:00 • committed by GitHub
parent ea9f35f082
commit c7b636e950
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
convert.py
View file

@ -913,12 +913,12 @@ class OutputFile:
scores.append(score) scores.append(score)
toktypes.append(toktype) toktypes.append(toktype)
if isinstance(vocab, SentencePieceVocab): if isinstance(vocab, SentencePieceVocab) or isinstance(vocab, HFVocab):
self.gguf.add_tokenizer_model("llama") self.gguf.add_tokenizer_model("llama")
elif isinstance(vocab, BpeVocab): elif isinstance(vocab, BpeVocab):
self.gguf.add_tokenizer_model("gpt2") self.gguf.add_tokenizer_model("gpt2")
else: else:
raise ValueError(f'Unknown vocab type: Not BpeVocab or SentencePieceVocab') raise ValueError(f'Unknown vocab type: Not BpeVocab or SentencePieceVocab or HFVocab')
self.gguf.add_token_list(tokens) self.gguf.add_token_list(tokens)
self.gguf.add_token_scores(scores) self.gguf.add_token_scores(scores)
self.gguf.add_token_types(toktypes) self.gguf.add_token_types(toktypes)