From 2897aa628c1c09fa8f1239a4ef86b1b02a509a9d Mon Sep 17 00:00:00 2001
From: Pierrick HYMBERT <pierrick.hymbert@gmail.com>
Date: Sun, 7 Apr 2024 23:47:26 +0200
Subject: [PATCH] llama: dbrx: revert

---
 llama.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llama.cpp b/llama.cpp
index de51aaf46..f6b8dacdd 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -7242,9 +7242,6 @@ struct llm_build_context {
                 cur = moe_out;
             }
 
-            cur = ggml_add(ctx0, cur, ffn_inp);
-            cb(cur, "ffn_out", il);
-
             // DbrxNormAttentionNorm
             {
                 cur = llm_build_norm(ctx0, cur, hparams,
@@ -7253,6 +7250,9 @@ struct llm_build_context {
                 cb(cur, "layer_out_norm", il);
             }
 
+            cur = ggml_add(ctx0, cur, ffn_inp);
+            cb(cur, "ffn_out", il);
+
             ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
             if (layer_dir != nullptr) {
                 cur = ggml_add(ctx0, cur, layer_dir);