vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

change cross_entropy_loss to output average over all rows

Browse source 
this helps keeping the loss and gradients in a sane range
This commit is contained in:
xaedes 2023-07-02 21:05:12 +02:00
parent 87febeec91
commit 51dc77092f
No known key found for this signature in database
GPG key ID: 30030EDD817EA2B1
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
ggml.c
View file

@ -14334,7 +14334,7 @@ static void ggml_compute_forward_cross_entropy_loss_f32(
if (ith == 0) {
float * dp = (float *) dst->data;
ggml_vec_sum_f32(nth, dp, sums);
dp[0] *= -1.0f;
dp[0] *= -1.0f / (float) nr;
}
return;
}
@ -14506,7 +14506,7 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(
ggml_vec_scale_f32(nc, ds0, sum);
ggml_vec_add1_f32(nc, ds0, ds0, eps);
ggml_vec_sub_f32(nc, ds0, ds0, s1);
ggml_vec_scale_f32(nc, ds0, d[0]);
ggml_vec_scale_f32(nc, ds0, d[0] / (float) nr);
#ifndef NDEBUG