From 0c26799e77758a643279ef661c1521604e7d3b12 Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Tue, 25 Jul 2023 22:51:01 +0800
Subject: [PATCH] a better default rms_norm_eps

---
 llama.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 765b9dba5..7607c70d8 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -1030,7 +1030,7 @@ static void llama_model_load_internal(
     auto & hparams = model.hparams;
 
     // TODO: read from file
-    hparams.f_rms_norm_eps = rms_norm_eps;
+    hparams.f_rms_norm_eps = 5e-6;
 
     {
         switch (hparams.n_layer) {