Revert back to GPT2 tokenizer
This commit is contained in:
parent
e743386728
commit
6cdb9c44ae
1 changed files with 1 additions and 5 deletions
|
@ -1059,11 +1059,7 @@ class PersimmonModel(Model):
|
|||
|
||||
class StableLMModel(Model):
|
||||
def set_vocab(self):
|
||||
if (self.dir_model / "tokenizer.json").is_file():
|
||||
self._set_vocab_gpt2()
|
||||
else:
|
||||
# StableLM 2 1.6B uses a vocab in a similar format to Qwen's vocab
|
||||
self._set_vocab_qwen()
|
||||
self._set_vocab_gpt2()
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hparams = self.hparams
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue