vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : adapt plamo to new ffn

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2023-12-27 17:24:34 +02:00
parent 278f3e99c2
commit 9174699f84
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
llama.cpp
View file

@ -3512,7 +3512,7 @@ static bool llm_load_tensors(
layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);
layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);
// AWQ ScaleActivation layer
layer.ffn_act = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, backend, false);
}
@ -5724,6 +5724,7 @@ struct llm_build_context {
model.layers[il].ffn_up, NULL,
model.layers[il].ffn_gate, NULL,
model.layers[il].ffn_down, NULL,
NULL,
LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
cb(cur, "ffn_out", il);
}