vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

server : bach has to be allocated for n_parallel sequences

Browse source
This commit is contained in:
Georgi Gerganov 2023-10-20 20:42:45 +03:00
parent 6b2437e32d
commit 113dd60005
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
examples/server/server.cpp
View file

@ -631,7 +631,9 @@ struct llama_server_context
LOG_TEE(" -> Slot %i - max context: %i\n", slot.id, max_ctx_per_slot);
slots.push_back(slot);
}
batch = llama_batch_init(n_ctx, 0, 1);
batch = llama_batch_init(n_ctx, 0, params.n_parallel);
// empty system prompt
system_prompt = "";
num_tokens_system = 0;