From 71ff763e0e4d9134f2eb7e3f5846e1c3d28de382 Mon Sep 17 00:00:00 2001
From: Joan Martinez <joan.fontanals.martinez@jina.ai>
Date: Mon, 22 Apr 2024 18:02:48 +0200
Subject: [PATCH] feat: set proper vocab settings

---
 convert-hf-to-gguf.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index 23e1606ed..4619c8095 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -2736,6 +2736,11 @@ class JinaBertModel(BertModel):
 
             yield name, data
 
+    def set_vocab(self, *args, **kwargs):
+        super().set_vocab()
+        self.gguf_writer.add_add_bos_token(True)
+        self.gguf_writer.add_add_eos_token(True)
+
 
 JinaBertForMaskedML = JinaBertModel