vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server: cap n_predict if not set to n_ctx_train

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-12 13:38:02 +02:00
parent 91c736015b
commit 6fd5ad597f
1 changed files with 12 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

12
examples/server/server.cpp
View file

@ -887,6 +887,18 @@ struct server_context {
}); });
slot.params.n_predict = slot.n_predict; slot.params.n_predict = slot.n_predict;
} }
if (slot.params.n_predict < 1 && slot.ga_n == 1) {
auto n_ctx_train = llama_n_ctx_train(model);
LOG_WARNING("n_predict is not set and self-context extend is disabled. Limiting generated tokens to n_ctx_train to avoid EOS-less generation infinite loop", {
{"params.n_predict", slot.params.n_predict},
{"slot.n_predict", slot.n_predict},
{"n_slots", params.n_parallel},
{"n_ctx", n_ctx},
{"n_ctx_train", n_ctx_train},
{"ga_n", slot.ga_n},
});
slot.params.n_predict = n_ctx_train;
}
// infill // infill
slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix); slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix);