vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama: dbrx: hardcode nn.LayerNorm epsilon

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-08 01:17:33 +02:00
parent 74e6d876f6
commit f8f97e74f9
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
llama.cpp
View file

@ -3986,6 +3986,7 @@ static void llm_load_hparams(
case LLM_ARCH_DBRX: case LLM_ARCH_DBRX:
{ {
ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv); ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);
hparams.f_norm_eps = 1.e-5; // REVIEW is that OK ? https://pytorch.org/docs/stable/generated/torch.nn.LayerNorm.html
switch (hparams.n_layer) { switch (hparams.n_layer) {
case 40: model.type = e_model::MODEL_132B; break; case 40: model.type = e_model::MODEL_132B; break;