vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix layer input for swin norm

Browse source
This commit is contained in:
nopperl 2024-07-18 10:26:34 +02:00
parent f40cd2073a
commit 15260c5ba8
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/llama.cpp
View file

@ -13838,12 +13838,14 @@ struct llm_build_context {
struct ggml_tensor * inpSA = inpL;
// norm
if (!hparams.swin_norm) {
if (hparams.swin_norm) {
cur = inpL;
} else {
cur = llm_build_norm(ctx0, inpL, hparams,
model.layers[il].attn_norm, NULL,
LLM_NORM_RMS, cb, il);
cb(cur, "attn_norm", il);
}
cb(cur, "attn_norm", il);
// self-attention
{