From 226c5eed4e0a19943c52aaef924f203d2f9a08cc Mon Sep 17 00:00:00 2001
From: Eddie-Wang <wangjinheng1120@163.com>
Date: Sun, 23 Jun 2024 15:58:30 +0000
Subject: [PATCH] fix bo

---
 llama.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 049bc81dd..c710ef82b 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -11861,7 +11861,7 @@ struct llm_build_context {
                 cb(Kcur, "Kcur", il);
 
                 cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf,
-                        nullptr, model.layers[il].bo,
+                        nullptr, nullptr,
                         Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
 
                 cur = llm_build_norm(ctx0, cur, hparams,
@@ -11871,6 +11871,9 @@ struct llm_build_context {
 
                 cur = ggml_mul_mat(ctx0, model.layers[il].wo, cur);
                 cur = ggml_mul(ctx0, cur, model.layers[il].wo_scale);
+                if (model.layers[il].bo) {
+                    cur = ggml_add(ctx0, cur, model.layers[il].bo);
+                }
                 cb(cur, "attn_o_out", il);
             }