ggml : fix q5_0
This commit is contained in:
Georgi Gerganov
parent
e5e7a24ee7
commit
67b079fc98
1 changed files with 55 additions and 61 deletions
|
|
@ -4611,6 +4611,9 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
const int qk = QK8_0;
|
||||
const int nb = n / qk;
|
||||
|
||||
int ib = 0;
|
||||
float sumf = 0;
|
||||
|
||||
assert(n % qk == 0);
|
||||
assert(qk == QK5_0);
|
||||
assert(nrc == 1);
|
||||
|
|
@ -4632,13 +4635,11 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
uint64_t tmp0[4];
|
||||
uint64_t tmp1[4];
|
||||
|
||||
assert(nb % 2 == 0); // TODO: handle odd nb
|
||||
|
||||
for (int i = 0; i < nb; i += 2) {
|
||||
const block_q5_0 * restrict x0 = &x[i];
|
||||
const block_q5_0 * restrict x1 = &x[i + 1];
|
||||
const block_q8_0 * restrict y0 = &y[i];
|
||||
const block_q8_0 * restrict y1 = &y[i + 1];
|
||||
for (; ib + 1 < nb; ib += 2) {
|
||||
const block_q5_0 * restrict x0 = &x[ib];
|
||||
const block_q5_0 * restrict x1 = &x[ib + 1];
|
||||
const block_q8_0 * restrict y0 = &y[ib];
|
||||
const block_q8_0 * restrict y1 = &y[ib + 1];
|
||||
|
||||
const uint8x16_t m4b = vdupq_n_u8(0x0F);
|
||||
|
||||
|
|
@ -4690,7 +4691,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
ggml_vdotq_s32(vdupq_n_s32(0), v0_1hf, v1_1h))), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d));
|
||||
}
|
||||
|
||||
*s = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
|
||||
sumf = vaddvq_f32(sumv0) + vaddvq_f32(sumv1);
|
||||
#elif defined(__wasm_simd128__)
|
||||
v128_t sumv = wasm_f32x4_splat(0.0f);
|
||||
|
||||
|
|
@ -4698,9 +4699,9 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
uint64_t tmp[4];
|
||||
|
||||
// TODO: check if unrolling this is better
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const block_q5_0 * restrict x0 = &x[i];
|
||||
const block_q8_0 * restrict y0 = &y[i];
|
||||
for (; ib < nb; ++ib) {
|
||||
const block_q5_0 * restrict x0 = &x[ib];
|
||||
const block_q8_0 * restrict y0 = &y[ib];
|
||||
|
||||
const v128_t m4b = wasm_i8x16_splat(0x0F);
|
||||
|
||||
|
|
@ -4750,23 +4751,23 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
wasm_f32x4_splat(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d))));
|
||||
}
|
||||
|
||||
*s = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3);
|
||||
#elif defined(__AVX2__)
|
||||
// Initialize accumulator with zeros
|
||||
__m256 acc = _mm256_setzero_ps();
|
||||
|
||||
// Main loop
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (; ib < nb; ++ib) {
|
||||
/* Compute combined scale for the block */
|
||||
const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
|
||||
const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d));
|
||||
|
||||
__m256i qx = bytes_from_nibbles_32(x[i].qs);
|
||||
__m256i bxhi = bytes_from_bits_32(x[i].qh);
|
||||
__m256i qx = bytes_from_nibbles_32(x[ib].qs);
|
||||
__m256i bxhi = bytes_from_bits_32(x[ib].qh);
|
||||
bxhi = _mm256_andnot_si256(bxhi, _mm256_set1_epi8((char)0xF0));
|
||||
qx = _mm256_or_si256(qx, bxhi);
|
||||
|
||||
__m256i qy = _mm256_loadu_si256((const __m256i *)y[i].qs);
|
||||
__m256i qy = _mm256_loadu_si256((const __m256i *)y[ib].qs);
|
||||
|
||||
const __m256 q = mul_sum_i8_pairs_float(qx, qy);
|
||||
|
||||
|
|
@ -4774,19 +4775,19 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
acc = _mm256_fmadd_ps(d, q, acc);
|
||||
}
|
||||
|
||||
*s = hsum_float_8(acc);
|
||||
sumf = hsum_float_8(acc);
|
||||
#elif defined(__AVX__)
|
||||
// Initialize accumulator with zeros
|
||||
__m256 acc = _mm256_setzero_ps();
|
||||
__m128i mask = _mm_set1_epi8((char)0xF0);
|
||||
|
||||
// Main loop
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (; ib < nb; ++ib) {
|
||||
/* Compute combined scale for the block */
|
||||
const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d));
|
||||
const __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d));
|
||||
|
||||
__m256i bx_0 = bytes_from_nibbles_32(x[i].qs);
|
||||
const __m256i bxhi = bytes_from_bits_32(x[i].qh);
|
||||
__m256i bx_0 = bytes_from_nibbles_32(x[ib].qs);
|
||||
const __m256i bxhi = bytes_from_bits_32(x[ib].qh);
|
||||
__m128i bxhil = _mm256_castsi256_si128(bxhi);
|
||||
__m128i bxhih = _mm256_extractf128_si256(bxhi, 1);
|
||||
bxhil = _mm_andnot_si128(bxhil, mask);
|
||||
|
|
@ -4797,7 +4798,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
bxh = _mm_or_si128(bxh, bxhih);
|
||||
bx_0 = MM256_SET_M128I(bxh, bxl);
|
||||
|
||||
const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[i].qs);
|
||||
const __m256i by_0 = _mm256_loadu_si256((const __m256i *)y[ib].qs);
|
||||
|
||||
const __m256 q = mul_sum_i8_pairs_float(bx_0, by_0);
|
||||
|
||||
|
|
@ -4805,7 +4806,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
acc = _mm256_add_ps(_mm256_mul_ps(d, q), acc);
|
||||
}
|
||||
|
||||
*s = hsum_float_8(acc);
|
||||
sumf = hsum_float_8(acc);
|
||||
#elif defined(__riscv_v_intrinsic)
|
||||
float sumf = 0.0;
|
||||
|
||||
|
|
@ -4820,8 +4821,8 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
vuint32m2_t vt_3 = __riscv_vsll_vx_u32m2(vt_2, 16, vl);
|
||||
vuint32m2_t vt_4 = __riscv_vadd_vx_u32m2(vt_1, 12, vl);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
memcpy(&qh, x[i].qh, sizeof(uint32_t));
|
||||
for (; ib < nb; ++ib) {
|
||||
memcpy(&qh, x[ib].qh, sizeof(uint32_t));
|
||||
|
||||
// ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
|
||||
vuint32m2_t xha_0 = __riscv_vand_vx_u32m2(vt_2, qh, vl);
|
||||
|
|
@ -4840,10 +4841,10 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
vuint8mf2_t xh_1 = __riscv_vncvt_x_x_w_u8mf2(xhc_1, vl);
|
||||
|
||||
// load
|
||||
vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[i].qs, vl);
|
||||
vuint8mf2_t tx = __riscv_vle8_v_u8mf2(x[ib].qs, vl);
|
||||
|
||||
vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[i].qs, vl);
|
||||
vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[i].qs+16, vl);
|
||||
vint8mf2_t y0 = __riscv_vle8_v_i8mf2(y[ib].qs, vl);
|
||||
vint8mf2_t y1 = __riscv_vle8_v_i8mf2(y[ib].qs+16, vl);
|
||||
|
||||
vuint8mf2_t x_at = __riscv_vand_vx_u8mf2(tx, 0x0F, vl);
|
||||
vuint8mf2_t x_lt = __riscv_vsrl_vx_u8mf2(tx, 0x04, vl);
|
||||
|
|
@ -4870,8 +4871,6 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined(__POWER9_VECTOR__)
|
||||
const vector signed char lowMask = vec_splats((signed char)0xF);
|
||||
const vector unsigned char v4 = vec_splats((unsigned char)4);
|
||||
|
|
@ -4879,27 +4878,27 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
vector float vsumf0 = vec_splats(0.0f);
|
||||
|
||||
#pragma GCC unroll 4
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
__builtin_prefetch(x[i].qs, 0, 1);
|
||||
__builtin_prefetch(y[i].qs, 0, 1);
|
||||
for (; ib < nb; ++ib) {
|
||||
__builtin_prefetch(x[ib].qs, 0, 1);
|
||||
__builtin_prefetch(y[ib].qs, 0, 1);
|
||||
|
||||
vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d));
|
||||
vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d));
|
||||
vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d));
|
||||
vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d));
|
||||
vector float vd = vec_mul(vxd, vyd);
|
||||
|
||||
vector signed long long aux64x2_0 = {(uint64_t)(table_b2b_1[x[i].qh[0]]), (uint64_t)(table_b2b_1[x[i].qh[1]])};
|
||||
vector signed long long aux64x2_1 = {(uint64_t)(table_b2b_1[x[i].qh[2]]), (uint64_t)(table_b2b_1[x[i].qh[3]])};
|
||||
vector signed long long aux64x2_0 = {(uint64_t)(table_b2b_1[x[ib].qh[0]]), (uint64_t)(table_b2b_1[x[ib].qh[1]])};
|
||||
vector signed long long aux64x2_1 = {(uint64_t)(table_b2b_1[x[ib].qh[2]]), (uint64_t)(table_b2b_1[x[ib].qh[3]])};
|
||||
|
||||
vector signed char qh0 = (vector signed char)aux64x2_0;
|
||||
vector signed char qh1 = (vector signed char)aux64x2_1;
|
||||
|
||||
vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs);
|
||||
vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs);
|
||||
|
||||
vector signed char q5x0 = vec_sub(vec_and (qxs, lowMask), qh0);
|
||||
vector signed char q5x1 = vec_sub(vec_sr(qxs, v4), qh1);
|
||||
|
||||
vector signed char q8y0 = vec_xl( 0, y[i].qs);
|
||||
vector signed char q8y1 = vec_xl( 16, y[i].qs);
|
||||
vector signed char q8y0 = vec_xl( 0, y[ib].qs);
|
||||
vector signed char q8y1 = vec_xl( 16, y[ib].qs);
|
||||
|
||||
vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0));
|
||||
vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1));
|
||||
|
|
@ -4914,23 +4913,23 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4));
|
||||
vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8));
|
||||
|
||||
*s = vec_extract(vsumf0, 0);
|
||||
sumf = vec_extract(vsumf0, 0);
|
||||
|
||||
#elif defined(__loongarch_asx)
|
||||
// Initialize accumulator with zeros
|
||||
__m256 acc = (__m256)__lasx_xvldi(0);
|
||||
|
||||
// Main loop
|
||||
for (int i = 0; i < nb; i++) {
|
||||
for (; ib < nb; ++ib) {
|
||||
/* Compute combined scale for the block */
|
||||
const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d)); //FIXME
|
||||
const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); //FIXME
|
||||
|
||||
__m256i qx = bytes_from_nibbles_32(x[i].qs);
|
||||
__m256i bxhi = bytes_from_bits_32(x[i].qh);
|
||||
__m256i qx = bytes_from_nibbles_32(x[ib].qs);
|
||||
__m256i bxhi = bytes_from_bits_32(x[ib].qh);
|
||||
bxhi = __lasx_xvandn_v(bxhi, __lasx_xvreplgr2vr_b((char)0xF0));
|
||||
qx = __lasx_xvor_v(qx, bxhi);
|
||||
|
||||
__m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0);
|
||||
__m256i qy = __lasx_xvld((const __m256i *)y[ib].qs, 0);
|
||||
|
||||
const __m256 q = mul_sum_i8_pairs_float(qx, qy);
|
||||
|
||||
|
|
@ -4938,15 +4937,11 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
acc = __lasx_xvfmadd_s(d, q, acc);
|
||||
}
|
||||
|
||||
*s = hsum_float_8(acc);
|
||||
|
||||
#else
|
||||
// scalar
|
||||
float sumf = 0.0;
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
sumf = hsum_float_8(acc);
|
||||
#endif
|
||||
for (; ib < nb; ++ib) {
|
||||
uint32_t qh;
|
||||
memcpy(&qh, x[i].qh, sizeof(qh));
|
||||
memcpy(&qh, x[ib].qh, sizeof(qh));
|
||||
|
||||
int sumi = 0;
|
||||
|
||||
|
|
@ -4954,17 +4949,16 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r
|
|||
const uint8_t xh_0 = ((qh & (1u << (j + 0 ))) >> (j + 0 )) << 4;
|
||||
const uint8_t xh_1 = ((qh & (1u << (j + 16))) >> (j + 12));
|
||||
|
||||
const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16;
|
||||
const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16;
|
||||
const int32_t x0 = ((x[ib].qs[j] & 0x0F) | xh_0) - 16;
|
||||
const int32_t x1 = ((x[ib].qs[j] >> 4) | xh_1) - 16;
|
||||
|
||||
sumi += (x0 * y[i].qs[j]) + (x1 * y[i].qs[j + qk/2]);
|
||||
sumi += (x0 * y[ib].qs[j]) + (x1 * y[ib].qs[j + qk/2]);
|
||||
}
|
||||
|
||||
sumf += (GGML_FP16_TO_FP32(x[i].d)*GGML_FP16_TO_FP32(y[i].d)) * sumi;
|
||||
sumf += (GGML_FP16_TO_FP32(x[ib].d)*GGML_FP16_TO_FP32(y[ib].d)) * sumi;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
#endif
|
||||
}
|
||||
|
||||
void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue