vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sample from residual distribution on draft accept failure

Browse source
This commit is contained in:
Minsoo Cheong 2024-02-22 13:50:30 +09:00
parent c1bad4a549
commit a9335a5c2a
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
examples/speculative/speculative.cpp
View file

@ -298,12 +298,12 @@ int main(int argc, char ** argv) {
if (!accept) {
// all drafted tokens were rejected
// sample from the target model
token_id = llama_sampling_sample(ctx_sampling, ctx_tgt, NULL, drafts[s_keep].i_batch_tgt[i_dft]);
LOG("all drafted tokens were rejected, sampling from residual distribution\n");
token_id = llama_sample_token(ctx_tgt, &dist_tgt);
llama_sampling_accept(ctx_sampling, ctx_tgt, token_id, true);
token_str = llama_token_to_piece(ctx_tgt, token_id);
}
} else {
// greedy verification