From f833b751757ef76f301068a12423eb88c8627740 Mon Sep 17 00:00:00 2001 From: Eve <139727413+netrunnereve@users.noreply.github.com> Date: Fri, 27 Dec 2024 15:07:34 -0500 Subject: [PATCH] go all the way --- .../vulkan-shaders/dequant_funcs.comp | 3 --- .../vulkan-shaders/mul_mat_vec_q6_k.comp | 16 ++++++++-------- ggml/src/ggml-vulkan/vulkan-shaders/types.comp | 8 -------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp index 14490f1a8..91bb8f8db 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.comp @@ -7,9 +7,6 @@ #if defined(A_TYPE_PACKED16) layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];}; #endif -#if defined(A_TYPE_PACKED16_FLAT) -layout (binding = 0) readonly buffer A_PACKED16_FLAT {A_TYPE_PACKED16_FLAT data_a_packed16_flat[];}; -#endif #if defined(A_TYPE_PACKED32) layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];}; #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp index d922cafe3..ccaa0486d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp @@ -10,7 +10,7 @@ layout (constant_id = 0) const uint BLOCK_SIZE = 32; layout (constant_id = 1) const uint NUM_ROWS = 1; shared FLOAT_TYPE tmpsh[NUM_ROWS][BLOCK_SIZE]; -shared block_q6_K_packed16_flat blkcache[BLOCK_SIZE/16]; +shared block_q6_K_packed16 blkcache[BLOCK_SIZE/16]; void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { uint a_offset, b_offset, d_offset; @@ -22,7 +22,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const uint it_size = gl_WorkGroupSize.x/16; const uint tid = gl_LocalInvocationID.x; const uint itid = tid%16; // 0...15 - const uint ix = tid/16; + const uint ix = tid/16; const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128... const uint v_in = itid - 8*v_im; // 0...15 or 0...7 @@ -58,24 +58,24 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { // cache full superblock into shared memory with coalesced reads // we assume 64 threads here! [[unroll]] for (int l = 0; (l < 4) && (i0 + l < num_blocks_per_row); ++l) { - blkcache[l].blkd[tid] = data_a_packed16_flat[ib0 + i0 + l].blkd[tid]; - // we read beyond the struct size but it looks like vulkan doesn't care? o_O + blkcache[l].ql[tid] = data_a_packed16[ib0 + i0 + l].ql[tid]; + // hacky method of reading beyond ql and the block struct size but it looks like vulkan doesn't care? o_O // this assumes that the struct is packed in continous 16 bit blocks to work - blkcache[l].blkd[64 + tid] = data_a_packed16_flat[ib0 + i0 + l].blkd[64 + tid]; + blkcache[l].ql[64 + tid] = data_a_packed16[ib0 + i0 + l].ql[64 + tid]; } barrier(); if (i >= num_blocks_per_row) continue; - uint32_t ql0_u32 = uint32_t(blkcache[ix].blkd[ql_offset / 2]) | (uint32_t(blkcache[ix].blkd[ql_offset / 2 + 1]) << 16); - uint32_t ql32_u32 = uint32_t(blkcache[ix].blkd[ql_offset / 2 + 16]) | (uint32_t(blkcache[ix].blkd[ql_offset / 2 + 17]) << 16); + uint32_t ql0_u32 = uint32_t(blkcache[ix].ql[ql_offset / 2]) | (uint32_t(blkcache[ix].ql[ql_offset / 2 + 1]) << 16); + uint32_t ql32_u32 = uint32_t(blkcache[ix].ql[ql_offset / 2 + 16]) | (uint32_t(blkcache[ix].ql[ql_offset / 2 + 17]) << 16); uint32_t ql0_u32_lo4 = ql0_u32 & 0x0F0F0F0F; uint32_t ql0_u32_hi4 = (ql0_u32 >> 4) & 0x0F0F0F0F; uint32_t ql32_u32_lo4 = ql32_u32 & 0x0F0F0F0F; uint32_t ql32_u32_hi4 = (ql32_u32 >> 4) & 0x0F0F0F0F; - uint32_t qh_u32 = uint32_t(blkcache[ix].blkd[64 + qh_offset / 2]) | (uint32_t(blkcache[ix].blkd[64 + qh_offset / 2 + 1]) << 16); + uint32_t qh_u32 = uint32_t(blkcache[ix].qh[qh_offset / 2]) | (uint32_t(blkcache[ix].qh[qh_offset / 2 + 1]) << 16); uint32_t qh0_u32 = (qh_u32 & 0x03030303) << 4; uint32_t qh2_u32 = (qh_u32 & 0x0C0C0C0C) << 2; uint32_t qh4_u32 = (qh_u32 & 0x30303030); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp index 397b2f225..eecc47f3a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.comp @@ -273,18 +273,10 @@ struct block_q6_K_packed16 float16_t d; }; -struct block_q6_K_packed16_flat -{ - uint16_t blkd[96]; - int8_t scales[16]; - float16_t d; -}; - #if defined(DATA_A_Q6_K) #define QUANT_K QUANT_K_Q6_K #define A_TYPE block_q6_K #define A_TYPE_PACKED16 block_q6_K_packed16 -#define A_TYPE_PACKED16_FLAT block_q6_K_packed16_flat #endif // IQuants