vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add missing kv clear in llama_beam_search

Browse source
This commit is contained in:
David Renshaw 2024-04-13 14:26:19 -04:00
parent 4bd0f93e4a
commit e969ada736
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama.cpp
View file

@ -13063,6 +13063,11 @@ struct llama_beam_search_data {
}
llama_logit_info logit_info(ctx);
std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams);
// Clear the kv slot so that other beams may try different tokens at this position. The llama_decode()
// call in loop() will conclusively fill in the kv slot once the beams converge at this position.
llama_kv_cache_seq_rm(ctx, 0, n_past, -1);
size_t i=0;
if (next_beams.size() < n_beams) {
for (; next_beams.size() < n_beams ; ++i) {