From 6cdb9c44ae0d560f2a58c9de4f9be8c950c183f2 Mon Sep 17 00:00:00 2001
From: Anas Ahouzi <112881240+aahouzi@users.noreply.github.com>
Date: Fri, 1 Mar 2024 06:37:23 -0800
Subject: [PATCH] Revert back to GPT2 tokenizer

---
 convert-hf-to-gguf.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index d3e8ec1f6..90351ec66 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -1059,11 +1059,7 @@ class PersimmonModel(Model):
 
 class StableLMModel(Model):
     def set_vocab(self):
-        if (self.dir_model / "tokenizer.json").is_file():
-            self._set_vocab_gpt2()
-        else:
-            # StableLM 2 1.6B uses a vocab in a similar format to Qwen's vocab
-            self._set_vocab_qwen()
+        self._set_vocab_gpt2()
 
     def set_gguf_parameters(self):
         hparams = self.hparams