From 5aaf24766aa7d85077816b7857f38d0d1d6b982b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 9 Oct 2024 11:01:53 +0300 Subject: [PATCH] llama : add infill sampler --- common/common.h | 2 ++ examples/server/server.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/common/common.h b/common/common.h index 5ca8fd391..2fb92ae14 100644 --- a/common/common.h +++ b/common/common.h @@ -117,6 +117,8 @@ struct common_sampler_params { float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities float dynatemp_range = 0.00f; // 0.0 = disabled float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler + float infill_p = 0.80f; + float infill_p_eog = 0.01f; int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size) float penalty_repeat = 1.00f; // 1.0 = disabled float penalty_freq = 0.00f; // 0.0 = disabled diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 3992108e7..e9621ba93 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -873,6 +873,8 @@ struct server_context { slot.sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z); slot.sparams.typ_p = json_value(data, "typical_p", default_sparams.typ_p); slot.sparams.temp = json_value(data, "temperature", default_sparams.temp); + slot.sparams.infill_p = json_value(data, "infill_p", default_sparams.infill_p); + slot.sparams.infill_p_eog = json_value(data, "infill_p_eog", default_sparams.infill_p_eog); slot.sparams.dynatemp_range = json_value(data, "dynatemp_range", default_sparams.dynatemp_range); slot.sparams.dynatemp_exponent = json_value(data, "dynatemp_exponent", default_sparams.dynatemp_exponent); slot.sparams.penalty_last_n = json_value(data, "repeat_last_n", default_sparams.penalty_last_n); @@ -1241,6 +1243,8 @@ struct server_context { {"xtc_threshold", slot.sparams.xtc_threshold}, {"tfs_z", slot.sparams.tfs_z}, {"typical_p", slot.sparams.typ_p}, + {"infill_p", slot.sparams.infill_p}, + {"infill_p_eog", slot.sparams.infill_p_eog}, {"repeat_last_n", slot.sparams.penalty_last_n}, {"repeat_penalty", slot.sparams.penalty_repeat}, {"presence_penalty", slot.sparams.penalty_present},