vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use sentencepiece tokenizer, or fall back to hfft.

Browse source
This commit is contained in:
Pedro Cuenca 2024-03-28 18:59:26 +01:00
parent 23ffda00df
commit 16ede02a47
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
convert-hf-to-gguf.py
View file

@ -1057,7 +1057,10 @@ class LlamaModel(Model):
model_arch = gguf.MODEL_ARCH.LLAMA model_arch = gguf.MODEL_ARCH.LLAMA
def set_vocab(self): def set_vocab(self):
self._set_vocab_llama_hf() try:
self. _set_vocab_sentencepiece()
except FileNotFoundError:
self._set_vocab_llama_hf()
def set_gguf_parameters(self): def set_gguf_parameters(self):
super().set_gguf_parameters() super().set_gguf_parameters()