vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

iq1_m: faster ARM_NEON dot product

Browse source 
11.65 t/s -> 14.9 t/s
This commit is contained in:
Iwan Kawrakow 2024-03-25 12:47:21 +01:00
parent f664692fa8
commit b1d1c26034
1 changed files with 10 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

16
ggml-quants.c
View file

@ -9763,12 +9763,16 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void
#if defined __ARM_NEON #if defined __ARM_NEON
const int8x8_t minus1 = vdup_n_s8(-1);
const int8x8_t plus1 = vdup_n_s8(+1);
const int32x4_t mask = vdupq_n_s32(0x7); const int32x4_t mask = vdupq_n_s32(0x7);
const int32x4_t mone = vdupq_n_s32(1); const int32x4_t mone = vdupq_n_s32(1);
const int32x4_t mzero = vdupq_n_s32(0); const int32x4_t mzero = vdupq_n_s32(0);
ggml_int8x16x4_t deltas;
deltas.val[0] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(+1));
deltas.val[1] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(+1));
deltas.val[2] = vcombine_s8(vdup_n_s8(+1), vdup_n_s8(-1));
deltas.val[3] = vcombine_s8(vdup_n_s8(-1), vdup_n_s8(-1));
ggml_int8x16x4_t q1b; ggml_int8x16x4_t q1b;
ggml_int8x16x4_t q8b; ggml_int8x16x4_t q8b;
ggml_int8x16x4_t delta; ggml_int8x16x4_t delta;
@ -9805,10 +9809,10 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void
const int32x4_t p2 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(mzero, q1b.val[3], q8b.val[3])); const int32x4_t p2 = vpaddq_s32(ggml_vdotq_s32(mzero, q1b.val[2], q8b.val[2]), ggml_vdotq_s32(mzero, q1b.val[3], q8b.val[3]));
const int32x4_t p12 = vpaddq_s32(p1, p2); const int32x4_t p12 = vpaddq_s32(p1, p2);
delta.val[0] = vcombine_s8(qh[0] & 0x08 ? minus1 : plus1, qh[0] & 0x80 ? minus1 : plus1); delta.val[0] = deltas.val[((qh[0] & 0x08) >> 3) | ((qh[0] & 0x80) >> 6)];
delta.val[1] = vcombine_s8(qh[1] & 0x08 ? minus1 : plus1, qh[1] & 0x80 ? minus1 : plus1); delta.val[1] = deltas.val[((qh[1] & 0x08) >> 3) | ((qh[1] & 0x80) >> 6)];
delta.val[2] = vcombine_s8(qh[2] & 0x08 ? minus1 : plus1, qh[2] & 0x80 ? minus1 : plus1); delta.val[2] = deltas.val[((qh[2] & 0x08) >> 3) | ((qh[2] & 0x80) >> 6)];
delta.val[3] = vcombine_s8(qh[3] & 0x08 ? minus1 : plus1, qh[3] & 0x80 ? minus1 : plus1); delta.val[3] = deltas.val[((qh[3] & 0x08) >> 3) | ((qh[3] & 0x80) >> 6)];
const int32x4_t p3 = vpaddq_s32(ggml_vdotq_s32(mzero, delta.val[0], q8b.val[0]), ggml_vdotq_s32(mzero, delta.val[1], q8b.val[1])); const int32x4_t p3 = vpaddq_s32(ggml_vdotq_s32(mzero, delta.val[0], q8b.val[0]), ggml_vdotq_s32(mzero, delta.val[1], q8b.val[1]));
const int32x4_t p4 = vpaddq_s32(ggml_vdotq_s32(mzero, delta.val[2], q8b.val[2]), ggml_vdotq_s32(mzero, delta.val[3], q8b.val[3])); const int32x4_t p4 = vpaddq_s32(ggml_vdotq_s32(mzero, delta.val[2], q8b.val[2]), ggml_vdotq_s32(mzero, delta.val[3], q8b.val[3]));