vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : add struct llama_kv_cache (wip) [no ci]

Browse source
This commit is contained in:
Georgi Gerganov 2025-01-13 14:13:11 +02:00
parent 178a7eb952
commit f78b396ee7
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
8 changed files with 428 additions and 415 deletions
Download patch file Download diff file Expand all files Collapse all files

6
common/common.cpp
View file

@ -952,7 +952,9 @@ struct common_init_result common_init_from_params(common_params & params) {
return iparams;
}
if (params.ctx_shift && !llama_kv_cache_can_shift(lctx)) {
llama_kv_cache * kv = llama_get_kv_cache(lctx);
if (params.ctx_shift && !llama_kv_cache_can_shift(kv)) {
LOG_WRN("%s: KV cache shifting is not supported for this model, disabling KV cache shifting\n", __func__);
params.ctx_shift = false;
}
@ -1057,7 +1059,7 @@ struct common_init_result common_init_from_params(common_params & params) {
if (llama_model_has_decoder(model)) {
llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch)));
}
llama_kv_cache_clear(lctx);
llama_kv_cache_clear(kv);
llama_synchronize(lctx);
llama_perf_context_reset(lctx);
}