vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server: fix infinite loop

Browse source
This commit is contained in:
Pierrick HYMBERT 2024-04-19 13:33:16 +02:00
parent 558f69083a
commit 5d64ffd837
1 changed files with 17 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

30
examples/server/server.cpp
View file

@ -887,18 +887,6 @@ struct server_context {
}); });
slot.params.n_predict = slot.n_predict; slot.params.n_predict = slot.n_predict;
} }
if (slot.params.n_predict < 1 && slot.ga_n == 1) {
auto n_ctx_train = llama_n_ctx_train(model);
LOG_WARNING("n_predict is not set and self-context extend is disabled. Limiting generated tokens to n_ctx_train to avoid EOS-less generation infinite loop", {
{"params.n_predict", slot.params.n_predict},
{"slot.n_predict", slot.n_predict},
{"n_slots", params.n_parallel},
{"n_ctx", n_ctx},
{"n_ctx_train", n_ctx_train},
{"ga_n", slot.ga_n},
});
slot.params.n_predict = n_ctx_train;
}
// infill // infill
slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix); slot.params.input_prefix = json_value(data, "input_prefix", default_params.input_prefix);
@ -2270,7 +2258,23 @@ struct server_context {
}); });
} }
if (!process_token(result, slot)) { auto n_ctx_train = llama_n_ctx_train(model);
bool stop_prediction = false;
if (slot.params.n_predict < 1 && slot.ga_n == 1 && slot.n_decoded >= n_ctx_train) {
LOG_WARNING("n_predict is not set and self-context extend is disabled. Limiting generated tokens to n_ctx_train to avoid EOS-less generation infinite loop", {
{"params.n_predict", slot.params.n_predict},
{"slot.n_predict", slot.n_predict},
{"slot.n_decoded", slot.n_decoded},
{"n_slots", params.n_parallel},
{"n_ctx", n_ctx},
{"n_ctx_train", n_ctx_train},
{"ga_n", slot.ga_n},
});
slot.truncated = true;
stop_prediction = true;
}
if (!process_token(result, slot) || stop_prediction) {
slot.release(); slot.release();
slot.print_timings(); slot.print_timings();
send_final_response(slot); send_final_response(slot);