From 02addabbae0a3b8904f939381397308b21b507c3 Mon Sep 17 00:00:00 2001
From: Georgi Gerganov <ggerganov@gmail.com>
Date: Sat, 9 Mar 2024 14:26:28 +0200
Subject: [PATCH] common : better normalize impl

---
 common/common.cpp | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/common/common.cpp b/common/common.cpp
index ec6fc277f..16ef4d7f7 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1854,15 +1854,14 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) {
 }
 
 void llama_embd_normalize(const float * inp, float * out, int n) {
-    float norm = 0;
+    double sum = 0.0;
     for (int i = 0; i < n; i++) {
-        norm += inp[i] * inp[i];
+        sum += inp[i] * inp[i];
     }
-    norm = sqrt(norm);
-    if (norm == 0) {
-        return;
-    }
-    norm = 1.0 / norm;
+    sum = sqrt(sum);
+
+    const float norm = sum > 0.0 ? 1.0f / sum : 0.0f;
+
     for (int i = 0; i < n; i++) {
         out[i] = inp[i] * norm;
     }