vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move constant out of loop

Browse source
This commit is contained in:
Howard Su 2023-04-01 01:51:44 +08:00
parent 8febfc73af
commit 41e8d2b434
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
llama.cpp
View file

@ -790,6 +790,7 @@ static bool llama_eval_internal(
memcpy(embd->data, tokens, N*ggml_element_size(embd));
struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.tok_embeddings, embd);
struct ggml_tensor * scale = ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head));
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * inpSA = inpL;
@ -850,7 +851,7 @@ static bool llama_eval_internal(
struct ggml_tensor * KQ_scaled =
ggml_scale_inplace(ctx0,
KQ,
ggml_new_f32(ctx0, 1.0f/sqrtf(float(n_embd)/n_head)));
scale);
// KQ_masked = mask_past(KQ_scaled)
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);