vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama: dbrx: fix last normalization

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-07 23:40:12 +02:00
parent 0ab1bae854
commit 830e46d7ae
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama.cpp
View file

@ -7242,6 +7242,9 @@ struct llm_build_context {
cur = moe_out; cur = moe_out;
} }
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);
// DbrxNormAttentionNorm // DbrxNormAttentionNorm
{ {
cur = llm_build_norm(ctx0, cur, hparams, cur = llm_build_norm(ctx0, cur, hparams,
@ -7250,9 +7253,6 @@ struct llm_build_context {
cb(cur, "layer_out_norm", il); cb(cur, "layer_out_norm", il);
} }
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "ffn_out", il);
ggml_tensor * layer_dir = lctx.cvec.tensor_for(il); ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
if (layer_dir != nullptr) { if (layer_dir != nullptr) {
cur = ggml_add(ctx0, cur, layer_dir); cur = ggml_add(ctx0, cur, layer_dir);