vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix top-p sampling to match the standard definition (smallest set that has probability mass at least p, not largest set with probability mass less than p)

Browse source
This commit is contained in:
Alex Renda 2023-06-20 14:38:13 -04:00
parent 049aa16b8c
commit 1e7755cfcb
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama.cpp
View file

@ -2006,8 +2006,8 @@ void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * can
cum_sum += candidates->data[i].p;
// Check if the running sum is greater than p or if we have kept at least min_keep tokens
if (cum_sum > p && i >= min_keep) {
last_idx = i;
if (cum_sum > p && i + 1 >= min_keep) {
last_idx = i + 1;
break;
}
}