vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sort before apply softmax

Browse source
This commit is contained in:
Xuan Son Nguyen 2024-12-12 13:47:43 +01:00
parent cc90cdbc33
commit 29c1495afa
1 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
examples/server/utils.hpp
View file

@ -705,6 +705,11 @@ static std::vector<llama_token_data> get_token_probabilities(llama_context * ctx
cur[token_id] = llama_token_data{token_id, logits[token_id], 0.0f};
}
// sort tokens by logits
std::sort(cur.begin(), cur.end(), [](const llama_token_data & a, const llama_token_data & b) {
return a.logit > b.logit;
});
// apply softmax
float max_l = cur[0].logit;
float cum_sum = 0.0f;
@ -717,10 +722,5 @@ static std::vector<llama_token_data> get_token_probabilities(llama_context * ctx
cur[i].p /= cum_sum;
}
// sort tokens by probability
std::sort(cur.begin(), cur.end(), [](const llama_token_data & a, const llama_token_data & b) {
return a.p > b.p;
});
return cur;
}