vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server: infinite loop, move in process_token

Browse source 
server: infinite loop: set stop limit to true
This commit is contained in:
Pierrick HYMBERT 2024-04-19 13:58:11 +02:00
parent 5d64ffd837
commit 1423fcece8
1 changed files with 20 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

37
examples/server/server.cpp
View file

@ -1208,6 +1208,25 @@ struct server_context {
LOG_VERBOSE("eos token found", {}); LOG_VERBOSE("eos token found", {});
} }
auto n_ctx_train = llama_n_ctx_train(model);
if (slot.params.n_predict < 1 && slot.ga_n == 1 && slot.n_decoded >= n_ctx_train) {
LOG_WARNING(
"n_predict is not set and self-context extend is disabled. Limiting generated tokens to n_ctx_train to avoid EOS-less generation infinite loop",
{
{ "id_slot", slot.id },
{ "params.n_predict", slot.params.n_predict },
{ "slot.n_predict", slot.n_predict },
{ "slot.n_decoded", slot.n_decoded },
{ "n_slots", params.n_parallel },
{ "n_ctx", n_ctx },
{ "n_ctx_train", n_ctx_train },
{ "ga_n", slot.ga_n },
});
slot.truncated = true;
slot.has_next_token = false; // stop prediction
slot.stopped_limit = true;
}
LOG_VERBOSE("next token", { LOG_VERBOSE("next token", {
{"id_slot", slot.id}, {"id_slot", slot.id},
{"id_task", slot.id_task}, {"id_task", slot.id_task},
@ -2258,23 +2277,7 @@ struct server_context {
}); });
} }
auto n_ctx_train = llama_n_ctx_train(model); if (!process_token(result, slot)) {
bool stop_prediction = false;
if (slot.params.n_predict < 1 && slot.ga_n == 1 && slot.n_decoded >= n_ctx_train) {
LOG_WARNING("n_predict is not set and self-context extend is disabled. Limiting generated tokens to n_ctx_train to avoid EOS-less generation infinite loop", {
{"params.n_predict", slot.params.n_predict},
{"slot.n_predict", slot.n_predict},
{"slot.n_decoded", slot.n_decoded},
{"n_slots", params.n_parallel},
{"n_ctx", n_ctx},
{"n_ctx_train", n_ctx_train},
{"ga_n", slot.ga_n},
});
slot.truncated = true;
stop_prediction = true;
}
if (!process_token(result, slot) || stop_prediction) {
slot.release(); slot.release();
slot.print_timings(); slot.print_timings();
send_final_response(slot); send_final_response(slot);