vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

constraint : add name API

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2024-09-04 15:37:51 +03:00
parent c024fe45b0
commit 1a0de0b781
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
5 changed files with 52 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

8
common/sampling.cpp
View file

@ -198,9 +198,11 @@ llama_token gpt_sampler_sample(struct gpt_sampler * gsmpl, struct llama_context
auto & grmr = gsmpl->grmr; auto & grmr = gsmpl->grmr;
auto & smpl = gsmpl->smpl; auto & smpl = gsmpl->smpl;
auto * cur_p = llama_sampler_get_candidates(smpl); const auto * logits = llama_get_logits_ith(ctx, idx);
llama_sampler_set_logits(smpl, llama_get_logits_ith(ctx, idx)); llama_sampler_set_logits(smpl, logits);
auto * cur_p = llama_sampler_get_candidates(smpl);
llama_constraint_apply(bias, cur_p); llama_constraint_apply(bias, cur_p);
llama_constraint_apply(pnlt, cur_p); llama_constraint_apply(pnlt, cur_p);
@ -223,7 +225,7 @@ llama_token gpt_sampler_sample(struct gpt_sampler * gsmpl, struct llama_context
} }
// if the token is not valid, sample again, first apply the grammar constraints and then sample // if the token is not valid, sample again, first apply the grammar constraints and then sample
llama_sampler_set_logits(smpl, llama_get_logits_ith(ctx, idx)); llama_sampler_set_logits(smpl, logits);
llama_constraint_apply(bias, cur_p); llama_constraint_apply(bias, cur_p);
llama_constraint_apply(pnlt, cur_p); llama_constraint_apply(pnlt, cur_p);

2
common/sampling.h
View file

@ -63,7 +63,7 @@ struct gpt_sampler_params {
// gpt_sampler extends llama_sampler with additional functionality: // gpt_sampler extends llama_sampler with additional functionality:
// //
// - grammar support // - grammar support
// - custom sampler logic based on the paramerters // - custom sampler logic based on the parameters
// //
struct gpt_sampler; struct gpt_sampler;

11
include/llama.h
View file

@ -386,11 +386,11 @@ extern "C" {
double t_start_ms; double t_start_ms;
double t_end_ms; double t_end_ms;
double t_load_ms; double t_load_ms;
double t_sampling_ms; double t_sampler_ms;
double t_p_eval_ms; double t_p_eval_ms;
double t_eval_ms; double t_eval_ms;
int32_t n_sampling; int32_t n_sampler;
int32_t n_p_eval; int32_t n_p_eval;
int32_t n_eval; int32_t n_eval;
}; };
@ -1025,8 +1025,7 @@ extern "C" {
// user code can implement the interface below in order to create custom llama_constraint // user code can implement the interface below in order to create custom llama_constraint
struct llama_constraint_i { struct llama_constraint_i {
// TODO: add name API const char * (*name) (const struct llama_constraint * cnstr); // can be NULL
void (*accept)( struct llama_constraint * cnstr, llama_token token); // can be NULL void (*accept)( struct llama_constraint * cnstr, llama_token token); // can be NULL
void (*apply) ( struct llama_constraint * cnstr, llama_token_data_array * cur_p); // required void (*apply) ( struct llama_constraint * cnstr, llama_token_data_array * cur_p); // required
void (*reset) ( struct llama_constraint * cnstr); // can be NULL void (*reset) ( struct llama_constraint * cnstr); // can be NULL
@ -1035,8 +1034,6 @@ extern "C" {
// TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph
//void (*apply_ggml) (struct llama_constraint * cnstr, ...); //void (*apply_ggml) (struct llama_constraint * cnstr, ...);
// TODO: add API to get timing stats
}; };
struct llama_constraint { struct llama_constraint {
@ -1044,7 +1041,7 @@ extern "C" {
llama_constraint_context_t ctx; llama_constraint_context_t ctx;
}; };
LLAMA_API struct llama_constraint * llama_constraint_init_softmax (); LLAMA_API struct llama_constraint * llama_constraint_init_softmax (void);
LLAMA_API struct llama_constraint * llama_constraint_init_top_k (int32_t k, int32_t min_keep); LLAMA_API struct llama_constraint * llama_constraint_init_top_k (int32_t k, int32_t min_keep);
LLAMA_API struct llama_constraint * llama_constraint_init_top_p (float p, int32_t min_keep); LLAMA_API struct llama_constraint * llama_constraint_init_top_p (float p, int32_t min_keep);
LLAMA_API struct llama_constraint * llama_constraint_init_min_p (float p, int32_t min_keep); LLAMA_API struct llama_constraint * llama_constraint_init_min_p (float p, int32_t min_keep);

51
src/llama-sampling.cpp
View file

@ -428,6 +428,7 @@ void llama_constraint_penalties_impl(
// softmax // softmax
static struct llama_constraint_i llama_constraint_softmax_i = { static struct llama_constraint_i llama_constraint_softmax_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "softmax"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * /*cnstr*/, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * /*cnstr*/, llama_token_data_array * cur_p) {
llama_constraint_softmax_impl(cur_p); llama_constraint_softmax_impl(cur_p);
@ -454,9 +455,10 @@ struct llama_constraint_context_top_k {
}; };
static struct llama_constraint_i llama_constraint_top_k_i = { static struct llama_constraint_i llama_constraint_top_k_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "top-k"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_top_k *) cnstr->ctx; const auto * ctx = (llama_constraint_context_top_k *) cnstr->ctx;
llama_constraint_top_k_impl(cur_p, ctx->k, ctx->min_keep); llama_constraint_top_k_impl(cur_p, ctx->k, ctx->min_keep);
}, },
/* .reset = */ nullptr, /* .reset = */ nullptr,
@ -466,7 +468,7 @@ static struct llama_constraint_i llama_constraint_top_k_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_top_k *) cnstr->ctx; delete (llama_constraint_context_top_k *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_top_k_impl(int32_t k, size_t min_keep) { struct llama_constraint * llama_constraint_init_top_k_impl(int32_t k, size_t min_keep) {
@ -489,9 +491,10 @@ struct llama_constraint_context_top_p {
}; };
static struct llama_constraint_i llama_constraint_top_p_i = { static struct llama_constraint_i llama_constraint_top_p_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "top-p"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_top_p *) cnstr->ctx; const auto * ctx = (llama_constraint_context_top_p *) cnstr->ctx;
llama_constraint_top_p_impl(cur_p, ctx->p, ctx->min_keep); llama_constraint_top_p_impl(cur_p, ctx->p, ctx->min_keep);
}, },
/* .reset = */ nullptr, /* .reset = */ nullptr,
@ -501,7 +504,7 @@ static struct llama_constraint_i llama_constraint_top_p_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_top_p *) cnstr->ctx; delete (llama_constraint_context_top_p *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_top_p_impl(float p, size_t min_keep) { struct llama_constraint * llama_constraint_init_top_p_impl(float p, size_t min_keep) {
@ -524,9 +527,10 @@ struct llama_constraint_context_min_p {
}; };
static struct llama_constraint_i llama_constraint_min_p_i = { static struct llama_constraint_i llama_constraint_min_p_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "min-p"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_min_p *) cnstr->ctx; const auto * ctx = (llama_constraint_context_min_p *) cnstr->ctx;
llama_constraint_min_p_impl(cur_p, ctx->p, ctx->min_keep); llama_constraint_min_p_impl(cur_p, ctx->p, ctx->min_keep);
}, },
/* .reset = */ nullptr, /* .reset = */ nullptr,
@ -536,7 +540,7 @@ static struct llama_constraint_i llama_constraint_min_p_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_min_p *) cnstr->ctx; delete (llama_constraint_context_min_p *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_min_p_impl(float p, size_t min_keep) { struct llama_constraint * llama_constraint_init_min_p_impl(float p, size_t min_keep) {
@ -559,9 +563,10 @@ struct llama_constraint_context_tail_free {
}; };
static struct llama_constraint_i llama_constraint_tail_free_i = { static struct llama_constraint_i llama_constraint_tail_free_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "tail-free"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_tail_free *) cnstr->ctx; const auto * ctx = (llama_constraint_context_tail_free *) cnstr->ctx;
llama_constraint_tail_free_impl(cur_p, ctx->z, ctx->min_keep); llama_constraint_tail_free_impl(cur_p, ctx->z, ctx->min_keep);
}, },
/* .reset = */ nullptr, /* .reset = */ nullptr,
@ -571,7 +576,7 @@ static struct llama_constraint_i llama_constraint_tail_free_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_tail_free *) cnstr->ctx; delete (llama_constraint_context_tail_free *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_tail_free_impl(float z, size_t min_keep) { struct llama_constraint * llama_constraint_init_tail_free_impl(float z, size_t min_keep) {
@ -594,9 +599,10 @@ struct llama_constraint_context_typical {
}; };
static struct llama_constraint_i llama_constraint_typical_i = { static struct llama_constraint_i llama_constraint_typical_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "typical"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_typical *) cnstr->ctx; const auto * ctx = (llama_constraint_context_typical *) cnstr->ctx;
llama_constraint_typical_impl(cur_p, ctx->p, ctx->min_keep); llama_constraint_typical_impl(cur_p, ctx->p, ctx->min_keep);
}, },
/* .reset = */ nullptr, /* .reset = */ nullptr,
@ -606,7 +612,7 @@ static struct llama_constraint_i llama_constraint_typical_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_typical *) cnstr->ctx; delete (llama_constraint_context_typical *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_typical_impl(float p, size_t min_keep) { struct llama_constraint * llama_constraint_init_typical_impl(float p, size_t min_keep) {
@ -628,9 +634,10 @@ struct llama_constraint_context_temp {
}; };
static struct llama_constraint_i llama_constraint_temp_i = { static struct llama_constraint_i llama_constraint_temp_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "temp"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_temp *) cnstr->ctx; const auto * ctx = (llama_constraint_context_temp *) cnstr->ctx;
llama_constraint_temp_impl(cur_p, ctx->temp); llama_constraint_temp_impl(cur_p, ctx->temp);
}, },
/* .reset = */ nullptr, /* .reset = */ nullptr,
@ -640,7 +647,7 @@ static struct llama_constraint_i llama_constraint_temp_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_temp *) cnstr->ctx; delete (llama_constraint_context_temp *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_temp_impl(float temp) { struct llama_constraint * llama_constraint_init_temp_impl(float temp) {
@ -663,9 +670,10 @@ struct llama_constraint_context_temp_ext {
}; };
static struct llama_constraint_i llama_constraint_temp_ext_i = { static struct llama_constraint_i llama_constraint_temp_ext_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "temp-ext"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_temp_ext *) cnstr->ctx; const auto * ctx = (llama_constraint_context_temp_ext *) cnstr->ctx;
if (ctx->delta > 0) { if (ctx->delta > 0) {
const float temp_min = std::max(0.0f, ctx->temp - ctx->delta); const float temp_min = std::max(0.0f, ctx->temp - ctx->delta);
const float temp_max = ctx->temp + ctx->delta; const float temp_max = ctx->temp + ctx->delta;
@ -682,7 +690,7 @@ static struct llama_constraint_i llama_constraint_temp_ext_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_temp_ext *) cnstr->ctx; delete (llama_constraint_context_temp_ext *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_temp_ext_impl(float temp, float delta, float exponent) { struct llama_constraint * llama_constraint_init_temp_ext_impl(float temp, float delta, float exponent) {
@ -708,14 +716,15 @@ struct llama_constraint_context_grammar {
}; };
static struct llama_constraint_i llama_constraint_grammar_i = { static struct llama_constraint_i llama_constraint_grammar_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "grammar"; },
/* .accept = */ [](struct llama_constraint * cnstr, llama_token token) { /* .accept = */ [](struct llama_constraint * cnstr, llama_token token) {
auto * ctx = (llama_constraint_context_grammar *) cnstr->ctx; const auto * ctx = (llama_constraint_context_grammar *) cnstr->ctx;
if (ctx->grammar) { if (ctx->grammar) {
llama_grammar_accept_impl(*ctx->grammar, token); llama_grammar_accept_impl(*ctx->grammar, token);
} }
}, },
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_grammar *) cnstr->ctx; const auto * ctx = (llama_constraint_context_grammar *) cnstr->ctx;
if (ctx->grammar) { if (ctx->grammar) {
llama_constraint_grammar_impl(cur_p, *ctx->grammar); llama_constraint_grammar_impl(cur_p, *ctx->grammar);
} }
@ -747,14 +756,14 @@ static struct llama_constraint_i llama_constraint_grammar_i = {
return result; return result;
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
auto * ctx = (llama_constraint_context_grammar *) cnstr->ctx; const auto * ctx = (llama_constraint_context_grammar *) cnstr->ctx;
if (ctx->grammar) { if (ctx->grammar) {
llama_grammar_free_impl(ctx->grammar); llama_grammar_free_impl(ctx->grammar);
} }
delete ctx; delete ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_grammar_impl(const struct llama_vocab & vocab, const char * grammar_str, const char * grammar_root) { struct llama_constraint * llama_constraint_init_grammar_impl(const struct llama_vocab & vocab, const char * grammar_str, const char * grammar_root) {
@ -799,6 +808,7 @@ struct llama_constraint_context_penalties {
}; };
static struct llama_constraint_i llama_constraint_penalties_i = { static struct llama_constraint_i llama_constraint_penalties_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "penalties"; },
/* .accept = */ [](struct llama_constraint * cnstr, llama_token token) { /* .accept = */ [](struct llama_constraint * cnstr, llama_token token) {
auto * ctx = (llama_constraint_context_penalties *) cnstr->ctx; auto * ctx = (llama_constraint_context_penalties *) cnstr->ctx;
ctx->prev.push_back(token); ctx->prev.push_back(token);
@ -855,7 +865,7 @@ static struct llama_constraint_i llama_constraint_penalties_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_penalties *) cnstr->ctx; delete (llama_constraint_context_penalties *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_penalties_impl(const struct llama_vocab & vocab, int32_t penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, bool penalize_nl, bool ignore_eos) { struct llama_constraint * llama_constraint_init_penalties_impl(const struct llama_vocab & vocab, int32_t penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, bool penalize_nl, bool ignore_eos) {
@ -888,6 +898,7 @@ struct llama_constraint_context_logit_bias {
}; };
static struct llama_constraint_i llama_constraint_logit_bias_i = { static struct llama_constraint_i llama_constraint_logit_bias_i = {
/* .name = */ [](const struct llama_constraint * /*cnstr*/) { return "logit-bias"; },
/* .accept = */ nullptr, /* .accept = */ nullptr,
/* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) { /* .apply = */ [](struct llama_constraint * cnstr, llama_token_data_array * cur_p) {
auto * ctx = (llama_constraint_context_logit_bias *) cnstr->ctx; auto * ctx = (llama_constraint_context_logit_bias *) cnstr->ctx;
@ -905,7 +916,7 @@ static struct llama_constraint_i llama_constraint_logit_bias_i = {
}, },
/* .free = */ [](struct llama_constraint * cnstr) { /* .free = */ [](struct llama_constraint * cnstr) {
delete (llama_constraint_context_logit_bias *) cnstr->ctx; delete (llama_constraint_context_logit_bias *) cnstr->ctx;
} },
}; };
struct llama_constraint * llama_constraint_init_logit_bias_impl( struct llama_constraint * llama_constraint_init_logit_bias_impl(

22
src/llama.cpp
View file

@ -20609,7 +20609,7 @@ int32_t llama_chat_apply_template(
// sampling // sampling
// //
struct llama_constraint * llama_constraint_init_softmax() { struct llama_constraint * llama_constraint_init_softmax(void) {
return llama_constraint_init_softmax_impl(); return llama_constraint_init_softmax_impl();
} }
@ -20849,22 +20849,22 @@ int llama_split_prefix(char * dest, size_t maxlen, const char * split_path, int
void llama_print_timings(struct llama_context * ctx, struct llama_sampler * smpl) { void llama_print_timings(struct llama_context * ctx, struct llama_sampler * smpl) {
const llama_timings timings = { const llama_timings timings = {
/*.t_start_ms =*/ 1e-3 * ctx->t_start_us, /*.t_start_ms =*/ 1e-3 * ctx->t_start_us,
/*.t_end_ms =*/ 1.00 * ggml_time_ms(), /*.t_end_ms =*/ 1.00 * ggml_time_ms(),
/*.t_load_ms =*/ 1e-3 * ctx->t_load_us, /*.t_load_ms =*/ 1e-3 * ctx->t_load_us,
/*.t_sampling_ms =*/ 1e-3 * (smpl ? smpl->t_sample_us : 0.0), /*.t_sampler_ms =*/ 1e-3 * (smpl ? smpl->t_sample_us : 0.0),
/*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us, /*.t_p_eval_ms =*/ 1e-3 * ctx->t_p_eval_us,
/*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us, /*.t_eval_ms =*/ 1e-3 * ctx->t_eval_us,
/*.n_sampling =*/ std::max(0, smpl ? smpl->n_sample : 0), /*.n_sampler =*/ std::max(0, smpl ? smpl->n_sample : 0),
/*.n_p_eval =*/ std::max(0, ctx->n_p_eval), /*.n_p_eval =*/ std::max(0, ctx->n_p_eval),
/*.n_eval =*/ std::max(1, ctx->n_eval), /*.n_eval =*/ std::max(1, ctx->n_eval),
}; };
LLAMA_LOG_INFO("\n"); LLAMA_LOG_INFO("\n");
LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, timings.t_load_ms); LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, timings.t_load_ms);
LLAMA_LOG_INFO("%s: sampling time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", LLAMA_LOG_INFO("%s: sampling time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",
__func__, timings.t_sampling_ms, timings.n_sampling, timings.t_sampling_ms / timings.n_sampling, 1e3 / timings.t_sampling_ms * timings.n_sampling); __func__, timings.t_sampler_ms, timings.n_sampler, timings.t_sampler_ms / timings.n_sampler, 1e3 / timings.t_sampler_ms * timings.n_sampler);
LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n", LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",
__func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval); __func__, timings.t_p_eval_ms, timings.n_p_eval, timings.t_p_eval_ms / timings.n_p_eval, 1e3 / timings.t_p_eval_ms * timings.n_p_eval);
LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n", LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",