vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

simple : add parallel decoding support

Browse source
This commit is contained in:
Georgi Gerganov 2023-09-20 13:06:34 +03:00
parent addae65fd4
commit b377bf2266
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
7 changed files with 187 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

6
common/common.cpp
View file

@ -956,11 +956,11 @@ llama_token llama_sample_token(
if (mirostat == 1) {
static float mirostat_mu = 2.0f * mirostat_tau;
const int mirostat_m = 100;
llama_sample_temperature(ctx, &cur_p, temp);
llama_sample_temp(ctx, &cur_p, temp);
id = llama_sample_token_mirostat(ctx, &cur_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
} else if (mirostat == 2) {
static float mirostat_mu = 2.0f * mirostat_tau;
llama_sample_temperature(ctx, &cur_p, temp);
llama_sample_temp(ctx, &cur_p, temp);
id = llama_sample_token_mirostat_v2(ctx, &cur_p, mirostat_tau, mirostat_eta, &mirostat_mu);
} else {
// Temperature sampling
@ -968,7 +968,7 @@ llama_token llama_sample_token(
llama_sample_tail_free (ctx, &cur_p, tfs_z, 1);
llama_sample_typical (ctx, &cur_p, typical_p, 1);
llama_sample_top_p (ctx, &cur_p, top_p, 1);
llama_sample_temperature(ctx, &cur_p, temp);
llama_sample_temp(ctx, &cur_p, temp);
{
const int n_top = 10;