vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Disable warmup under MPI

Browse source
This commit is contained in:
Branden Butler 2023-09-25 17:41:57 -05:00
parent 4829c6224e
commit 16eff5af69
1 changed files with 7 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

7
common/common.cpp
View file

@ -1402,10 +1402,17 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_par
{ {
LOG("warming up the model with an empty run\n"); LOG("warming up the model with an empty run\n");
#ifndef GGML_USE_MPI
// When using MPI, llama_eval() enters into an infinite loop
// on non-head nodes. Thus, we only want to warmup the model here
// if we aren't using MPI.
// FIXME have a way to terminate the infinite loop so we can warmup the model
// in MPI mode
std::vector<llama_token> tmp = { llama_token_bos(model), llama_token_eos(model), }; std::vector<llama_token> tmp = { llama_token_bos(model), llama_token_eos(model), };
llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0)); llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0));
llama_kv_cache_clear(lctx); llama_kv_cache_clear(lctx);
llama_synchronize(lctx); llama_synchronize(lctx);
#endif
llama_reset_timings(lctx); llama_reset_timings(lctx);
} }