diff --git a/llama.cpp b/llama.cpp
index 765b9dba5..7607c70d8 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -1030,7 +1030,7 @@ static void llama_model_load_internal(
     auto & hparams = model.hparams;
 
     // TODO: read from file
-    hparams.f_rms_norm_eps = rms_norm_eps;
+    hparams.f_rms_norm_eps = 5e-6;
 
     {
         switch (hparams.n_layer) {