vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

batched-bench : add tensor_split param

Browse source
This commit is contained in:
Georgi Gerganov 2024-01-05 10:31:15 +02:00 • committed by slaren
parent a1ab35c682
commit 1fa7ee2e51
1 changed files with 3 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
examples/batched-bench/batched-bench.cpp
View file

@ -88,7 +88,10 @@ int main(int argc, char ** argv) {
llama_model_params model_params = llama_model_default_params();
const std::vector<float> t_split (LLAMA_MAX_DEVICES, 0.0f);
model_params.n_gpu_layers = n_gpu_layers;
model_params.tensor_split = t_split.data();
llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params);