vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swap arguments to vDSP_vdiv call

Browse source 
documentation for vDSP_vdiv states: "Note that B comes before A!"
This commit is contained in:
xaedes 2023-05-08 20:09:52 +02:00 • committed by Georgi Gerganov
parent cafbb785fa
commit 9c3fe4eb76
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 7 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

14
ggml.c
View file

@ -9831,15 +9831,15 @@ static void ggml_compute_forward_rms_norm_back_f32(
sum_xdz += (ggml_float)(x[i00] * dz[i00]); sum_xdz += (ggml_float)(x[i00] * dz[i00]);
} }
const float mean = sum_xx/ne00; const ggml_float mean = sum_xx/ne00;
const float mean_eps = sum_xx/ne00 + eps; const ggml_float mean_eps = sum_xx/ne00 + eps;
const float sum_eps = sum_xx + eps*ne00; const ggml_float sum_eps = sum_xx + eps*ne00;
const float mean_xdz = sum_xdz/ne00; const ggml_float mean_xdz = sum_xdz/ne00;
// we could cache rms from forward pass to improve performance. // we could cache rms from forward pass to improve performance.
// to do this implement ggml_rms and compose ggml_rms_norm using ggml_rms. // to do this implement ggml_rms and compose ggml_rms_norm using ggml_rms.
const float rms = sqrtf(mean_eps); const ggml_float rms = sqrtf(mean_eps);
const float rrms = 1.0f / sqrtf(mean_eps); const ggml_float rrms = 1.0f / sqrtf(mean_eps);
const float scale = -rrms/(ne00 * mean_eps); // -1/(n*rms**3) const ggml_float scale = -rrms/(ne00 * mean_eps); // -1/(n*rms**3)
{ {
// z = rms_norm(x) // z = rms_norm(x)