diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index 23e1606ed..4619c8095 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -2736,6 +2736,11 @@ class JinaBertModel(BertModel):
 
             yield name, data
 
+    def set_vocab(self, *args, **kwargs):
+        super().set_vocab()
+        self.gguf_writer.add_add_bos_token(True)
+        self.gguf_writer.add_add_eos_token(True)
+
 
 JinaBertForMaskedML = JinaBertModel