From ecbfb1b5842a033c422ec76f9c37e391bad98178 Mon Sep 17 00:00:00 2001
From: Pierrick Hymbert <pierrick.hymbert@gmail.com>
Date: Fri, 12 Apr 2024 21:41:14 +0200
Subject: [PATCH] Wrong input was being fed to moe layer. This needs to be
 corrected

Co-authored-by: Megha Agarwal <16129366+megha95@users.noreply.github.com>
---
 llama.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 00a849b54..000346f45 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -7165,7 +7165,7 @@ struct llm_build_context {
 
             // feed-forward network
             // MoE branch
-            cur = llm_build_norm(ctx0, cur, hparams,
+            cur = llm_build_norm(ctx0, ffn_inp, hparams,
                                  model.layers[il].attn_out_norm, NULL,
                                  LLM_NORM, cb, il);
             cb(cur, "attn_out_norm", il);