From 89475fb320168e0a82f19e74285748f843106242 Mon Sep 17 00:00:00 2001
From: xaedes <xaedes@gmail.com>
Date: Sun, 28 May 2023 22:40:58 +0200
Subject: [PATCH] slightly improve how cross entropy loss is compute

btw: directly implemented cross entropy loss seems to have way lower magnitudes than when implemented with softmax and log.
probably the input to log gets closer to zero due to float numerics.
maybe the multiplication by (1.0-eps)/sum is more accurate..
---
 ggml.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ggml.c b/ggml.c
index bdd29ac50..b75d55c3d 100644
--- a/ggml.c
+++ b/ggml.c
@@ -12961,10 +12961,10 @@ static void ggml_compute_forward_cross_entropy_loss_f32(
             }
 
             assert(sum > 0.0);
-            sum = 1.0/sum;
+            // sum = 1.0/sum;
         }
         // avoid log(0) by rescaling from [0..1] to [eps..1]
-        sum = sum * (1.0f - eps);
+        sum = (1.0f - eps) / sum;
         ggml_vec_scale_f32(nc, st, sum);
         ggml_vec_add1_f32(nc, st, st, eps);
         ggml_vec_log_f32(nc, st, st);