vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix nasty bug in ggml_compute_forward_mul_mat_f32() and reenable BLAS

Browse source
This commit is contained in:
Georgi Gerganov 2023-03-25 16:09:54 +02:00
parent e899bf54b2
commit 2a2e63ce05
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
2 changed files with 40 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama.cpp
View file

@ -917,8 +917,7 @@ static bool llama_eval_internal(
struct ggml_tensor * KQ_scaled =
ggml_scale(ctx0,
KQ,
ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head))
);
ggml_new_f32(ctx0, 1.0f/sqrt(float(n_embd)/n_head)));
// KQ_masked = mask_past(KQ_scaled)
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ_scaled, n_past);
@ -934,7 +933,7 @@ static bool llama_eval_internal(
ggml_view_1d(ctx0, kv_self.v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(kv_self.v)*n_embd),
n_embd/n_head, n_head, n_past + N),
1, 2, 0, 3),
ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
ggml_new_tensor_3d(ctx0, kv_self.v->type, n_past + N, n_embd/n_head, n_head));
// KQV = transpose(V) * KQ_soft_max
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);