vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

batched : add bench tool

Browse source
This commit is contained in:
Georgi Gerganov 2023-10-08 15:57:16 +03:00
parent 8e6716a102
commit bf06d654de
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
4 changed files with 205 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/batched/batched.cpp
View file

@ -66,7 +66,7 @@ int main(int argc, char ** argv) {
ctx_params.seed = 1234;
ctx_params.n_ctx = n_kv_req;
ctx_params.n_batch = std::max(n_len, n_parallel);
ctx_params.n_threads = params.n_threads;
ctx_params.n_threads = params.n_threads;
ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch;
llama_context * ctx = llama_new_context_with_model(model, ctx_params);