vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added docs explaining the rollback op

Browse source
This commit is contained in:
l3utterfly 2024-01-31 10:47:24 +09:00
parent d5f4dec935
commit 7fe612dc73
1 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
common/sampling.h
View file

@ -121,6 +121,10 @@ void llama_sampling_accept(
llama_token id, llama_token id,
bool apply_grammar); bool apply_grammar);
// this performs rollback of the latest sampling operation by "rollback_num" tokens;
// it simply strikes the latest "rollback_num" tokens from the "prev" vector
// in general, the rollback is "imperfect", meaning the "forgotten tokens" which were dropped when the length of "prev" exceeded "n_prev" cannot be recalled after rollback
// however, if `sampling_params.n_prev` >= `sampling_params.penalty_last_n` + `rollback_num`, then it becomes "perfect" rollback
void llama_sampling_rollback( void llama_sampling_rollback(
struct llama_sampling_context * ctx_sampling, struct llama_sampling_context * ctx_sampling,
int rollback_num); int rollback_num);