vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama: dbrx: no attention output layer

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-06 22:22:57 +02:00
parent 76f266beef
commit 9c7dedb0f3
1 changed files with 47 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama.cpp
View file

@ -7165,7 +7165,7 @@ struct llm_build_context {
cb(Vcur, "Vcur", il);
cur = llm_build_kv(ctx0, model, hparams, kv_self, gf,
model.layers[il].wo, model.layers[il].bo,
NULL, NULL,
Kcur, Vcur, Qcur, KQ_mask, nullptr, n_ctx, n_tokens, kv_head, n_kv, 1.0f, cb, il);
}
@ -7182,6 +7182,7 @@ struct llm_build_context {
// feed-forward network
// MoE branch
{
// FIXME REVIEW: I do not see this op in https://huggingface.co/databricks/dbrx-instruct/blob/464e701f50aef4c1b59c81fb5667819a5d08e108/modeling_dbrx.py#L727
cur = llm_build_norm(ctx0, ffn_inp, hparams,
NULL, NULL,
@ -7212,7 +7213,6 @@ struct llm_build_context {
// compute expert outputs
ggml_tensor * moe_out = nullptr;
for (int i = 0; i < n_expert_used; ++i) {
ggml_tensor * cur_expert;
@ -7244,6 +7244,7 @@ struct llm_build_context {
}
}
cur = moe_out;
}
// DbrxNormAttentionNorm
{