vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[ggml-aarch64] use intrinsics in q4_0_4_4 gemv

Browse source
This commit is contained in:
Shupei Fan 2024-10-25 14:07:17 +08:00
parent a9e8a9a030
commit 102299e30c
1 changed files with 42 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

96
ggml/src/ggml-aarch64.c
View file

@ -620,62 +620,50 @@ void ggml_gemv_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void *
#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)
if (ggml_cpu_has_neon()) { if (ggml_cpu_has_neon()) {
const void * b_ptr = vx; const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
const void * a_ptr = vy;
float * res_ptr = s; float * res_ptr = s;
__asm__ __volatile__( for (int x = 0; x < nc / ncols_interleaved; x++) {
"movi v31.16b, #0x4\n" const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
"movi v30.16b, #0xf0\n"
"add %x[b_ptr], %x[b_ptr], #0x8\n" float32x4_t sumf = vdupq_n_f32(0);
"1:" // Column loop for (int l = 0; l < nb; l++) {
"add x22, %x[a_ptr], #0x2\n" uint8x16_t b_0 = vld1q_u8(b_ptr[l].qs + 0);
"movi v29.16b, #0x0\n" uint8x16_t b_1 = vld1q_u8(b_ptr[l].qs + 16);
"mov x21, %x[nb]\n" uint8x16_t b_2 = vld1q_u8(b_ptr[l].qs + 32);
"2:" // Block loop uint8x16_t b_3 = vld1q_u8(b_ptr[l].qs + 48);
"ldr q28, [%x[b_ptr], #0x0]\n"
"ldr q27, [x22, #0x0]\n" int8x16_t b_0_hi = vreinterpretq_s8_u8(b_0 & 0xF0);
"movi v26.4s, #0x0\n" int8x16_t b_0_lo = vreinterpretq_s8_u8(b_0 << 4);
"sub x20, x22, #0x2\n" int8x16_t b_1_hi = vreinterpretq_s8_u8(b_1 & 0xF0);
"ldr q25, [x22, #0x10]\n" int8x16_t b_1_lo = vreinterpretq_s8_u8(b_1 << 4);
"ldr q24, [%x[b_ptr], #0x10]\n" int8x16_t b_2_hi = vreinterpretq_s8_u8(b_2 & 0xF0);
"sub x21, x21, #0x1\n" int8x16_t b_2_lo = vreinterpretq_s8_u8(b_2 << 4);
"add x22, x22, #0x22\n" int8x16_t b_3_hi = vreinterpretq_s8_u8(b_3 & 0xF0);
"ldr q23, [%x[b_ptr], #0x20]\n" int8x16_t b_3_lo = vreinterpretq_s8_u8(b_3 << 4);
"ldr q22, [%x[b_ptr], #0x30]\n"
"ld1r { v21.8h }, [x20]\n" int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 0);
"ldr q20, [%x[b_ptr], #-0x8]\n" int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16);
"sshl v16.16b, v28.16b, v31.16b\n"
"and v28.16b, v28.16b, v30.16b\n" int32x4_t sumi = vdupq_n_s32(0);
"sshl v19.16b, v24.16b, v31.16b\n" sumi = vdotq_laneq_s32(sumi, b_0_lo, a_0, 0);
"and v24.16b, v24.16b, v30.16b\n" sumi = vdotq_laneq_s32(sumi, b_0_hi, a_1, 0);
"add %x[b_ptr], %x[b_ptr], #0x48\n" sumi = vdotq_laneq_s32(sumi, b_1_lo, a_0, 1);
"sshl v18.16b, v23.16b, v31.16b\n" sumi = vdotq_laneq_s32(sumi, b_1_hi, a_1, 1);
"and v23.16b, v23.16b, v30.16b\n" sumi = vdotq_laneq_s32(sumi, b_2_lo, a_0, 2);
".inst 0x4f9be21a // sdot v26.4s, v16.16b, v27.4b[0]\n" sumi = vdotq_laneq_s32(sumi, b_2_hi, a_1, 2);
"sshl v17.16b, v22.16b, v31.16b\n" sumi = vdotq_laneq_s32(sumi, b_3_lo, a_0, 3);
"and v22.16b, v22.16b, v30.16b\n" sumi = vdotq_laneq_s32(sumi, b_3_hi, a_1, 3);
"fcvtl v21.4s, v21.4h\n"
"fcvtl v16.4s, v20.4h\n" float32x4_t a_d = vcvt_f32_f16(vld1_dup_f16((const float16_t *)&a_ptr[l].d));
".inst 0x4f99e39a // sdot v26.4s, v28.16b, v25.4b[0]\n" float32x4_t b_d = vcvt_f32_f16(vld1_f16((const float16_t *)b_ptr[l].d));
"fmul v16.4s, v16.4s, v21.4s\n" float32x4_t d = a_d * b_d;
".inst 0x4fbbe27a // sdot v26.4s, v19.16b, v27.4b[1]\n"
".inst 0x4fb9e31a // sdot v26.4s, v24.16b, v25.4b[1]\n" sumf = vmlaq_f32(sumf, d, vcvtq_n_f32_s32(sumi, 4));
".inst 0x4f9bea5a // sdot v26.4s, v18.16b, v27.4b[2]\n" }
".inst 0x4f99eafa // sdot v26.4s, v23.16b, v25.4b[2]\n"
".inst 0x4fbbea3a // sdot v26.4s, v17.16b, v27.4b[3]\n" vst1q_f32(res_ptr + x * 4, sumf);
".inst 0x4fb9eada // sdot v26.4s, v22.16b, v25.4b[3]\n" }
"scvtf v26.4s, v26.4s, #0x4\n"
"fmla v29.4s, v26.4s, v16.4s\n"
"cbnz x21, 2b\n"
"sub %x[nc], %x[nc], #0x4\n"
"str q29, [%x[res_ptr], #0x0]\n"
"add %x[res_ptr], %x[res_ptr], #0x10\n"
"cbnz %x[nc], 1b\n"
: [b_ptr] "+&r" (b_ptr), [res_ptr] "+&r" (res_ptr), [nc] "+&r" (nc)
: [a_ptr] "r" (a_ptr), [nb] "r" (nb)
: "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22"
);
return; return;
} }
#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) #endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)