vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add warm up also for promp_len=32, warm up both gemm and gemv

Browse source
This commit is contained in:
luoyu-intel 2024-07-18 15:04:41 +08:00
parent 6e3ab8736e
commit 90e93db548
1 changed files with 19 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

19
common/common.cpp
View file

@ -2126,6 +2126,25 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_par
llama_kv_cache_clear(lctx);
llama_synchronize(lctx);
llama_reset_timings(lctx);
tmp.clear();
for (int i = 0; i < 32; i++)
{
tmp.push_back(bos);
}
tmp.push_back(eos);
if (llama_model_has_encoder(model)) {
llama_encode(lctx, llama_batch_get_one(tmp.data(), tmp.size(), 0, 0));
llama_token decoder_start_token_id = llama_model_decoder_start_token(model);
if (decoder_start_token_id == -1) {
decoder_start_token_id = bos;
}
tmp.clear();
tmp.push_back(decoder_start_token_id);
}
llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0));
llama_kv_cache_clear(lctx);
llama_synchronize(lctx);
llama_reset_timings(lctx);
}
return std::make_tuple(model, lctx);