From d7fed4732f73bb3bb499b4972464796793a4939d Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:24:00 +0800 Subject: [PATCH] fix for typical sampler --- llama.cpp | 6 +++++- otherarch/llama_v3.cpp | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 92980c0d7..3cffc7201 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1234,6 +1234,7 @@ struct llama_model_loader { // determine file type based on the number of tensors for each quantization and print meta data // TODO: make optional + if(false) //disable this log for now { std::map n_type; @@ -3974,7 +3975,10 @@ void llama_sample_typical(struct llama_context * ctx, llama_token_data_array * c float entropy = 0.0f; for (size_t i = 0; i < candidates->size; ++i) { - entropy += -candidates->data[i].p * logf(candidates->data[i].p); + if(candidates->data[i].p>0) + { + entropy += -candidates->data[i].p * logf(candidates->data[i].p); + } } // Compute the absolute difference between negative log probability and entropy for each candidate diff --git a/otherarch/llama_v3.cpp b/otherarch/llama_v3.cpp index 0ce016fc3..609b2bfa3 100644 --- a/otherarch/llama_v3.cpp +++ b/otherarch/llama_v3.cpp @@ -2663,7 +2663,10 @@ void llama_v3_sample_typical(struct llama_v3_context * ctx, llama_v3_token_data_ float entropy = 0.0f; for (size_t i = 0; i < candidates->size; ++i) { - entropy += -candidates->data[i].p * logf(candidates->data[i].p); + if(candidates->data[i].p>0) + { + entropy += -candidates->data[i].p * logf(candidates->data[i].p); + } } // Compute the absolute difference between negative log probability and entropy for each candidate