From b0e4ccbeb95f3987052fbc08500459773216a691 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Sat, 4 Jan 2025 14:57:40 -0500 Subject: [PATCH] revert it --- .../vulkan-shaders/mul_mat_vec_q2_k.comp | 16 +++--- .../vulkan-shaders/mul_mat_vec_q3_k.comp | 12 ++--- .../vulkan-shaders/mul_mat_vec_q4_k.comp | 49 ++++++------------- .../vulkan-shaders/mul_mat_vec_q5_k.comp | 12 ++--- 4 files changed, 34 insertions(+), 55 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp index 934213446..921759bc8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp @@ -14,13 +14,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { // 16 threads are used to process each block const uint it_size = gl_WorkGroupSize.x/16; const uint tid = gl_LocalInvocationID.x; - const uint itid = tid%16; // 0...16 - const uint ix = tid/16; + const uint itid = tid%16; // 0...15 + const uint ix = tid/16; - const uint step = 8; - - const uint v_im = itid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const uint v_in = itid - step*v_im; // 0...15 or 0...7 + const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128... + const uint v_in = itid - 8*v_im; // 0...15 or 0...7 const uint l0 = 2*v_in; // 0...15 const uint q_offset = 32*v_im + l0; @@ -44,7 +42,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const FLOAT_TYPE dall = d.x; const FLOAT_TYPE dmin = d.y; - uint32_t s0_u32 = data_a_packed32[ib0 + i].scales[s_offset / 4 + 0]; + uint32_t s0_u32 = data_a_packed32[ib0 + i].scales[s_offset / 4]; uint32_t s4_u32 = data_a_packed32[ib0 + i].scales[s_offset / 4 + 1]; uint32_t s0_lo4_u32 = s0_u32 & 0x0F0F0F0F; @@ -57,13 +55,13 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { uvec4 s0_hi4 = uvec4(unpack8(s0_hi4_u32)); uvec4 s4_hi4 = uvec4(unpack8(s4_hi4_u32)); - uint16_t qs0_u16 = data_a_packed16[ib0 + i].qs[q_offset / 2 + 0]; + uint16_t qs0_u16 = data_a_packed16[ib0 + i].qs[q_offset / 2]; uint16_t qs16_u16 = data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]; uvec2 qs0 = uvec2(unpack8(qs0_u16)); uvec2 qs16 = uvec2(unpack8(qs16_u16)); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]; + B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2]; B_TYPE_VEC2 b16 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]; B_TYPE_VEC2 b32 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]; B_TYPE_VEC2 b48 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp index 86b0159d9..b9a87f13a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp @@ -14,13 +14,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { // 16 threads are used to process each block const uint it_size = gl_WorkGroupSize.x/16; const uint tid = gl_LocalInvocationID.x; - const uint itid = tid%16; // 0...16 - const uint ix = tid/16; + const uint itid = tid%16; // 0...15 + const uint ix = tid/16; - const uint step = 8; - - const uint v_im = itid/step; // 0 or 1. 0 computes 0..., 1 computes 128... - const uint v_in = itid - step*v_im; // 0...15 or 0...7 + const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128... + const uint v_in = itid - 8*v_im; // 0...15 or 0...7 const uint8_t m = uint8_t(1 << (4 * v_im)); @@ -60,7 +58,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { - B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]; + B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2]; B_TYPE_VEC2 b16 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]; B_TYPE_VEC2 b32 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]; B_TYPE_VEC2 b48 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp index ee439f5c4..225f0ce70 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp @@ -6,8 +6,6 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; -shared FLOAT_TYPE sc[BLOCK_SIZE/16][2][8]; - void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { uint a_offset, b_offset, d_offset; get_offsets(a_offset, b_offset, d_offset); @@ -27,16 +25,6 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 const uint v_in = il % 2; - // v_im may be incorrect - const uint itid8 = itid%8; - const uint a = (itid8 > 5) ? 4 : 0; - const uint b = (itid8 > 3) ? 0x0f : 0x3f; - const uint c = (itid8 > 3) ? 0xc0 : 0xFF; - const uint dd = (itid8 > 3) ? 2 : 0; - const uint gg = (itid8 > 1) ? 1 : 0; - const uint g = (itid8 > 3) ? 8 : gg; - const uint h = itid8%2; - const uint l0 = n * (2 * ir + v_in); // 0...15 const uint q_offset = 32*v_im + l0; const uint y_offset = 64*v_im + l0; @@ -62,23 +50,18 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ]; const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2]; const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4]; + uvec2 scale0 = uvec2(unpack8(scale0_u32)); + uvec2 scale4 = uvec2(unpack8(scale4_u32)); + uvec2 scale8 = uvec2(unpack8(scale8_u32)); - uvec2 scale[3]; - scale[0] = uvec2(unpack8(scale0_u32)); - scale[1] = uvec2(unpack8(scale4_u32)); - scale[2] = uvec2(unpack8(scale8_u32)); - - sc[ix][v_im][itid8] = FLOAT_TYPE(((scale[gg][h] >> a) & b) | ((scale[g][h] & c) >> dd)); - barrier(); - - //const uint32_t sc0 = ( scale0.x & 0x3f); - //const uint32_t sc1 = ( scale0.y & 0x3f); - //const uint32_t sc2 = ( scale4.x & 0x3f); - //const uint32_t sc3 = ( scale4.y & 0x3f); - //const uint32_t sc4 = (( scale8.x & 0x0f) | ((scale0.x & 0xc0) >> 2)); - //const uint32_t sc5 = (( scale8.y & 0x0f) | ((scale0.y & 0xc0) >> 2)); - //const uint32_t sc6 = (((scale8.x >> 4) & 0x0f) | ((scale4.x & 0xc0) >> 2)); - //const uint32_t sc7 = (((scale8.y >> 4) & 0x0f) | ((scale4.y & 0xc0) >> 2)); + const uint32_t sc0 = ( scale0.x & 0x3f); + const uint32_t sc1 = ( scale0.y & 0x3f); + const uint32_t sc2 = ( scale4.x & 0x3f); + const uint32_t sc3 = ( scale4.y & 0x3f); + const uint32_t sc4 = (( scale8.x & 0x0f) | ((scale0.x & 0xc0) >> 2)); + const uint32_t sc5 = (( scale8.y & 0x0f) | ((scale0.y & 0xc0) >> 2)); + const uint32_t sc6 = (((scale8.x >> 4) & 0x0f) | ((scale4.x & 0xc0) >> 2)); + const uint32_t sc7 = (((scale8.y >> 4) & 0x0f) | ((scale4.y & 0xc0) >> 2)); uint32_t qs0_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4]; uint32_t qs64_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4 + 16]; @@ -121,11 +104,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const FLOAT_TYPE sz = fma(FLOAT_TYPE(by20.x), q4_8, fma(FLOAT_TYPE(by20.y), q4_9, fma(FLOAT_TYPE(by20.z), q4_10, FLOAT_TYPE(by20.w) * q4_11))); const FLOAT_TYPE sw = fma(FLOAT_TYPE(by232.x), q4_12, fma(FLOAT_TYPE(by232.y), q4_13, fma(FLOAT_TYPE(by232.z), q4_14, FLOAT_TYPE(by232.w) * q4_15))); const FLOAT_TYPE smin = - fma(FLOAT_TYPE(by10.x), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.x), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.x), sc[ix][v_im][6], fma(FLOAT_TYPE(by232.x), sc[ix][v_im][7], - fma(FLOAT_TYPE(by10.y), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.y), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.y), sc[ix][v_im][6], fma(FLOAT_TYPE(by232.y), sc[ix][v_im][7], - fma(FLOAT_TYPE(by10.z), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.z), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.z), sc[ix][v_im][6], fma(FLOAT_TYPE(by232.z), sc[ix][v_im][7], - fma(FLOAT_TYPE(by10.w), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.w), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.w), sc[ix][v_im][6], FLOAT_TYPE(by232.w) * sc[ix][v_im][7]))))))))))))))); - temp[j][n] = fma(dall, fma(sx, sc[ix][v_im][0], fma(sy, sc[ix][v_im][1], fma(sz, sc[ix][v_im][4], sw * sc[ix][v_im][5]))), fma(-dmin, smin, temp[j][n])); + fma(FLOAT_TYPE(by10.x), sc2, fma(FLOAT_TYPE(by132.x), sc3, fma(FLOAT_TYPE(by20.x), sc6, fma(FLOAT_TYPE(by232.x), sc7, + fma(FLOAT_TYPE(by10.y), sc2, fma(FLOAT_TYPE(by132.y), sc3, fma(FLOAT_TYPE(by20.y), sc6, fma(FLOAT_TYPE(by232.y), sc7, + fma(FLOAT_TYPE(by10.z), sc2, fma(FLOAT_TYPE(by132.z), sc3, fma(FLOAT_TYPE(by20.z), sc6, fma(FLOAT_TYPE(by232.z), sc7, + fma(FLOAT_TYPE(by10.w), sc2, fma(FLOAT_TYPE(by132.w), sc3, fma(FLOAT_TYPE(by20.w), sc6, FLOAT_TYPE(by232.w) * sc7))))))))))))))); + temp[j][n] = fma(dall, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dmin, smin, temp[j][n])); } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp index 0a68891c3..bea1dc860 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp @@ -15,8 +15,8 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { // 16 threads are used to process each block const uint it_size = gl_WorkGroupSize.x/16; const uint tid = gl_LocalInvocationID.x; - const uint itid = tid%16; // 0...16 - const uint ix = tid/16; + const uint itid = tid%16; // 0...15 + const uint ix = tid/16; const uint il = itid/4; // 0...3 const uint ir = itid - 4*il; // 0...7 or 0...3 @@ -49,9 +49,9 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ]; uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2]; uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4]; - uvec4 scale0 = uvec4(unpack8(scale0_u32)); - uvec4 scale4 = uvec4(unpack8(scale4_u32)); - uvec4 scale8 = uvec4(unpack8(scale8_u32)); + uvec2 scale0 = uvec2(unpack8(scale0_u32)); + uvec2 scale4 = uvec2(unpack8(scale4_u32)); + uvec2 scale8 = uvec2(unpack8(scale8_u32)); const uint32_t sc0 = ( scale0.x & 0x3f); const uint32_t sc1 = ( scale0.y & 0x3f); @@ -74,7 +74,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { uint32_t qs0_16_lo4_offset16 = ((qh >> (2*v_im)) & 0x01010101) << 4; uint32_t qs0_16_hi4_offset16 = ((qh >> (2*v_im)) & 0x02020202) << 3; - uint32_t qs64_80_lo4_offset16 = ((qh >> (2*v_im)) & 0x10101010) << 0; + uint32_t qs64_80_lo4_offset16 = ((qh >> (2*v_im)) & 0x10101010); uint32_t qs64_80_hi4_offset16 = ((qh >> (2*v_im)) & 0x20202020) >> 1; qs0_16_u32_lo4 += qs0_16_lo4_offset16;