vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : add missing kv clear in llama_beam_search (#6664)

Browse source
This commit is contained in:
David Renshaw 2024-04-14 15:24:15 -04:00 • committed by GitHub
parent 04fbc5f23e
commit 1958f7e06c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama.cpp
View file

@ -13063,6 +13063,11 @@ struct llama_beam_search_data {
} }
llama_logit_info logit_info(ctx); llama_logit_info logit_info(ctx);
std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams); std::vector<llama_token_data> next_tokens = logit_info.top_k(n_beams);
// Clear the kv slot so that other beams may try different tokens at this position. The llama_decode()
// call in loop() will conclusively fill in the kv slot once the beams converge at this position.
llama_kv_cache_seq_rm(ctx, 0, n_past, -1);
size_t i=0; size_t i=0;
if (next_beams.size() < n_beams) { if (next_beams.size() < n_beams) {
for (; next_beams.size() < n_beams ; ++i) { for (; next_beams.size() < n_beams ; ++i) {