diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index 882327a30..5b5a1c255 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -2340,6 +2340,7 @@ class Gemma2Model(Model):
 
     def set_vocab(self):
         self._set_vocab_llama_hf()
+        self.gguf_writer.add_add_space_prefix(False)
 
     def set_gguf_parameters(self):
         hparams = self.hparams