From 51dc77092fa0aaaf832dbfda46058a413521b8a9 Mon Sep 17 00:00:00 2001
From: xaedes <xaedes@gmail.com>
Date: Sun, 2 Jul 2023 21:05:12 +0200
Subject: [PATCH] change cross_entropy_loss to output average over all rows

this helps keeping the loss and gradients in a sane range
---
 ggml.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ggml.c b/ggml.c
index d718de33b..07d100bf0 100644
--- a/ggml.c
+++ b/ggml.c
@@ -14334,7 +14334,7 @@ static void ggml_compute_forward_cross_entropy_loss_f32(
         if (ith == 0) {
             float * dp = (float *) dst->data;
             ggml_vec_sum_f32(nth, dp, sums);
-            dp[0] *= -1.0f;
+            dp[0] *= -1.0f / (float) nr;
         }
         return;
     }
@@ -14506,7 +14506,7 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32(
         ggml_vec_scale_f32(nc, ds0, sum);
         ggml_vec_add1_f32(nc, ds0, ds0, eps);
         ggml_vec_sub_f32(nc, ds0, ds0, s1);
-        ggml_vec_scale_f32(nc, ds0, d[0]);
+        ggml_vec_scale_f32(nc, ds0, d[0] / (float) nr);
 
 
 #ifndef NDEBUG