vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Measure eval time only for single-token calls

Browse source
This commit is contained in:
Georgi Gerganov 2023-03-22 07:22:51 +02:00
parent 71ed3d224d
commit a9f900b645
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
llama.cpp
View file

@ -836,8 +836,11 @@ static bool llama_eval_internal(
ggml_free(ctx0); ggml_free(ctx0);
lctx.t_eval_us += ggml_time_us() - t_start_us; // measure the performance only for the single-token evals
lctx.n_eval++; if (N == 1) {
lctx.t_eval_us += ggml_time_us() - t_start_us;
lctx.n_eval++;
}
return true; return true;
} }