vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sampling : hide prev behind API and apply #3661

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2023-10-20 18:26:20 +03:00
parent 7e2b5fb1dd
commit 56ba00b923
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
9 changed files with 119 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/parallel/parallel.cpp
View file

@ -330,7 +330,7 @@ int main(int argc, char ** argv) {
const llama_token id = llama_sampling_sample(client.ctx_sampling, ctx, NULL, client.i_batch - i);
llama_sampling_accept(client.ctx_sampling, ctx, id);
llama_sampling_accept(client.ctx_sampling, ctx, id, true);
if (client.n_decoded == 1) {
// start measuring generation time after the first token to make sure all concurrent clients