From 830e46d7ae1c0190bbb5e5f58fc3932a58222998 Mon Sep 17 00:00:00 2001
From: Pierrick HYMBERT <pierrick.hymbert@gmail.com>
Date: Sun, 7 Apr 2024 23:40:12 +0200
Subject: [PATCH] llama: dbrx:  fix last normalization

---
 llama.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llama.cpp b/llama.cpp
index f6b8dacdd..de51aaf46 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -7242,6 +7242,9 @@ struct llm_build_context {
                 cur = moe_out;
             }
 
+            cur = ggml_add(ctx0, cur, ffn_inp);
+            cb(cur, "ffn_out", il);
+
             // DbrxNormAttentionNorm
             {
                 cur = llm_build_norm(ctx0, cur, hparams,
@@ -7250,9 +7253,6 @@ struct llm_build_context {
                 cb(cur, "layer_out_norm", il);
             }
 
-            cur = ggml_add(ctx0, cur, ffn_inp);
-            cb(cur, "ffn_out", il);
-
             ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
             if (layer_dir != nullptr) {
                 cur = ggml_add(ctx0, cur, layer_dir);