From 16ede02a474df3ba2d89faed036420fe66ab51b9 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Thu, 28 Mar 2024 18:59:26 +0100 Subject: [PATCH] Use sentencepiece tokenizer, or fall back to hfft. --- convert-hf-to-gguf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 6bc210100..c1fe385f3 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1057,7 +1057,10 @@ class LlamaModel(Model): model_arch = gguf.MODEL_ARCH.LLAMA def set_vocab(self): - self._set_vocab_llama_hf() + try: + self. _set_vocab_sentencepiece() + except FileNotFoundError: + self._set_vocab_llama_hf() def set_gguf_parameters(self): super().set_gguf_parameters()