vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remember and restore kv cache data pointers

Browse source 
because their value is stored in buf and overwritten by memcpy
This commit is contained in:
xaedes 2023-04-14 02:38:47 +02:00
parent 3d59769c3b
commit 283156c1fd
No known key found for this signature in database
GPG key ID: 30030EDD817EA2B1
1 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama.cpp
View file

@ -2092,7 +2092,11 @@ void llama_set_kv_cache(
int n_token_count) {
// Make sure we have the same kv cache setup
LLAMA_ASSERT(ctx->model.kv_self.buf.size == n_size);
void * k_data = ctx->model.kv_self.k->data; // remember data pointers
void * v_data = ctx->model.kv_self.v->data; // because their value is stored in buf and overwritten by memcpy
memcpy(ctx->model.kv_self.buf.addr, kv_cache, n_size);
ctx->model.kv_self.k->data = k_data; // restore correct data pointers
ctx->model.kv_self.v->data = v_data;
ctx->model.kv_self.n = n_token_count;
}