vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add RMS norm and use it (#187)

Browse source 
* add ggml_rms_norm

* update op num
This commit is contained in:
hoangmit 2023-03-15 18:41:38 -04:00 • committed by GitHub
parent 27944c4206
commit 6eac39ba95
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 134 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

6
main.cpp
View file

@ -588,7 +588,7 @@ bool llama_eval(
// norm
{
cur = ggml_norm(ctx0, inpL);
cur = ggml_rms_norm(ctx0, inpL);
// cur = attention_norm*cur
cur = ggml_mul(ctx0,
@ -678,7 +678,7 @@ bool llama_eval(
{
// norm
{
cur = ggml_norm(ctx0, inpFF);
cur = ggml_rms_norm(ctx0, inpFF);
// cur = ffn_norm*cur
cur = ggml_mul(ctx0,
@ -713,7 +713,7 @@ bool llama_eval(
// norm
{
inpL = ggml_norm(ctx0, inpL);
inpL = ggml_rms_norm(ctx0, inpL);
// inpL = norm*inpL
inpL = ggml_mul(ctx0,