vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Simplified chances calculation

Browse source 
To be more inline with the original implementation, chance is calculated once at the beginning.
This commit is contained in:
MaggotHATE 2024-10-04 18:30:46 +05:00 • committed by GitHub
parent 9455194056
commit db54ac5df4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/llama-sampling.cpp
View file

@ -1089,18 +1089,15 @@ static void llama_sample_xtc_apply(struct llama_sampler * smpl, llama_token_data
// going through all candidates from back to front, easier to keep the last of probables // going through all candidates from back to front, easier to keep the last of probables
for (int i = (cur_p->size - 1); i >= 0; --i) { for (int i = (cur_p->size - 1); i >= 0; --i) {
if (cur_p->data[i].p >= ctx->threshold && cur_p->data[i].p <= ctx->threshold_max) { if (cur_p->data[i].p >= ctx->threshold && cur_p->data[i].p <= ctx->threshold_max) {
if (removed == 0 || chance <= ctx->probability) { ++removed;
++removed; if (removed > 1) {
if (removed >= 2) { // .logits are used for sorting and calculating .p in llama_sample_softmax_impl
// .logits are used for sorting and calculating .p in llama_sample_softmax_impl cur_p->data[i].logit = -999.0f;
cur_p->data[i].logit = -999.0f;
chance = (float)(rd()%100)/100;
}
} }
} }
} }
if (removed >= 2) { if (removed > 1) {
// sorting with new logits, ex-last probable will be the first anyway // sorting with new logits, ex-last probable will be the first anyway
std::sort(cur_p->data, cur_p->data + cur_p->size, [](const llama_token_data & a, const llama_token_data & b) { std::sort(cur_p->data, cur_p->data + cur_p->size, [](const llama_token_data & a, const llama_token_data & b) {
return a.logit > b.logit; return a.logit > b.logit;