vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Revert back to GPT2 tokenizer

Browse source
This commit is contained in:
Anas Ahouzi 2024-03-01 06:37:23 -08:00
parent e743386728
commit 6cdb9c44ae
1 changed files with 1 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
convert-hf-to-gguf.py
View file

@ -1059,11 +1059,7 @@ class PersimmonModel(Model):
class StableLMModel(Model): class StableLMModel(Model):
def set_vocab(self): def set_vocab(self):
if (self.dir_model / "tokenizer.json").is_file(): self._set_vocab_gpt2()
self._set_vocab_gpt2()
else:
# StableLM 2 1.6B uses a vocab in a similar format to Qwen's vocab
self._set_vocab_qwen()
def set_gguf_parameters(self): def set_gguf_parameters(self):
hparams = self.hparams hparams = self.hparams