From 4547848743ac7dd7ccbb15c7afa50ab0a892be60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ma=C3=ABl=20Kerbiriou?= <m431.kerbiriou@gmail.com>
Date: Mon, 20 Mar 2023 12:15:13 +0100
Subject: [PATCH] log llama's entropy

---
 utils.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/utils.cpp b/utils.cpp
index c71ffa8e7..f1a73aa74 100644
--- a/utils.cpp
+++ b/utils.cpp
@@ -521,18 +521,26 @@ struct SoftMaxSampler {
 
         // compute probs for the tokens
         double sum_p = 0.0;
+        double sum_act = 0.0;
+        double entropy = 0.0;
         for (const auto & kv : logits_id) {
+            sum_act += kv.first;
             double logp = kv.first - maxl;
             double p = exp(logp);
             probs.push_back(p);
             sum_p += p;
+            entropy -= p * logp;
         }
 
         // normalize the probs
         const double scale = 1.0 / sum_p;
+        entropy = entropy * scale + log(sum_p);
         for (auto & p : probs) {
             p *= scale;
         }
+
+        // Scaled activations stats & distribution info
+        logprintf( "%s: top_sact=%f mean_sact=%f top_p=%f entropy=%f\n", __func__, logits_id[0].first, sum_act / n, probs[0], entropy);
     }