vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama: dbrx: load norm eps in hparams

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-08 14:38:21 +02:00
parent 506cc2ea53
commit eb0847e6b1
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
llama.cpp
View file

@ -3985,7 +3985,8 @@ static void llm_load_hparams(
} break;
case LLM_ARCH_DBRX:
{
ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);
switch (hparams.n_layer) {
case 40: model.type = e_model::MODEL_132B; break;