vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tests for min_p, sampling queue (#5147)

Browse source
This commit is contained in:
Johannes Gäßler 2024-01-28 09:35:14 +01:00 • committed by GitHub
parent af4980bfed
commit b2b2bf988c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 159 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama.cpp
View file

@ -8133,6 +8133,11 @@ void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * c
}
void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int32_t k, size_t min_keep) {
// TODO: move bucket sort to separate function so that top_p/tail_free/typical/softmax first is equally fast
// if (k >= (int32_t)candidates->size) {
// return;
// }
const int64_t t_start_sample_us = ggml_time_us();
k = std::max(k, (int) min_keep);