diff --git a/llama.cpp b/llama.cpp
index f9a8efee1..4818fe6c7 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -1030,7 +1030,7 @@ static void llama_model_load_internal(
     auto & hparams = model.hparams;
 
     // TODO: read from file
-    hparams.f_rms_norm_eps = 5e-6;
+    hparams.f_rms_norm_eps = rms_norm_eps;
 
     {
         switch (hparams.n_layer) {