vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Minor change of the constant names for minicpm

Browse source
This commit is contained in:
vincent 2024-02-07 08:47:04 +08:00
parent a9353ecd97
commit 390996d84c
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
llama.cpp
View file

@ -6830,8 +6830,8 @@ struct llm_build_context {
const int64_t n_embd = hparams.n_embd; const int64_t n_embd = hparams.n_embd;
//TODO: if the model varies, these parameters need to be read from the model //TODO: if the model varies, these parameters need to be read from the model
const int scale_emb = 12; const int64_t n_embd_base = 256;
const int dim_model_base = 256; const float scale_embd = 12.0f;
const float scale_depth = 1.4f; const float scale_depth = 1.4f;
struct ggml_tensor * cur; struct ggml_tensor * cur;
@ -6841,7 +6841,7 @@ struct llm_build_context {
cb(inpL, "inp_embd", -1); cb(inpL, "inp_embd", -1);
// scale the input embeddings // scale the input embeddings
inpL = ggml_scale(ctx0, inpL, float(scale_emb)); inpL = ggml_scale(ctx0, inpL, scale_embd);
cb(inpL, "inp_scaled", -1); cb(inpL, "inp_scaled", -1);
// inp_pos - contains the positions // inp_pos - contains the positions
@ -6953,7 +6953,7 @@ struct llm_build_context {
cb(cur, "result_norm", -1); cb(cur, "result_norm", -1);
// lm_head scaling // lm_head scaling
const float scale_lmhead = float(dim_model_base)/float(n_embd); const float scale_lmhead = float(n_embd_base)/float(n_embd);
cur = ggml_scale(ctx0, cur, scale_lmhead); cur = ggml_scale(ctx0, cur, scale_lmhead);
cb(cur, "lmhead_scaling", -1); cb(cur, "lmhead_scaling", -1);