vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : add llama_kv_cache_shift_seq + no more context swaps

Browse source
This commit is contained in:
Georgi Gerganov 2023-09-18 18:00:25 +03:00
parent 86c90e34f5
commit 0cbf3bfef8
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
4 changed files with 66 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

1
common/common.cpp
View file

@ -781,6 +781,7 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_par
std::vector<llama_token> tmp = { llama_token_bos(lctx), llama_token_eos(lctx), };
llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0), params.n_threads);
llama_kv_cache_keep_seq(lctx, -1);
llama_reset_timings(lctx);
}