vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix q4_0/q4_1 mmv, 65 -> 49 failures

Browse source
This commit is contained in:
Jared Van Bortel 2024-01-24 11:56:43 -05:00
parent 0787b80db8
commit 1a14099c43
6 changed files with 92 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

1
CMakeLists.txt
View file

@ -464,6 +464,7 @@ if (LLAMA_KOMPUTE)
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${source} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${source}
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/common.comp ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/common.comp
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_getrows.comp ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_getrows.comp
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n.comp ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n.comp
COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${CMAKE_CURRENT_SOURCE_DIR}/${source} COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${CMAKE_CURRENT_SOURCE_DIR}/${source}
COMMENT "Compiling ${source} to ${spv_file}" COMMENT "Compiling ${source} to ${spv_file}"

88
ggml-kompute.cpp
View file

@ -1003,32 +1003,40 @@ static void ggml_vk_mul_mat_mat_f32(kp::Sequence& seq,
seq.record<kp::OpAlgoDispatch>(s_algo); seq.record<kp::OpAlgoDispatch>(s_algo);
} }
static void ggml_vk_mul_mat_q4_x( static void ggml_vk_mul_mat_impl(
const std::vector<uint32_t>& spirv, const char * suffix, uint32_t block_size, kp::Sequence& seq, const std::vector<uint32_t>& spirv, const char * suffix, uint32_t block_size, kp::Sequence& seq,
const std::shared_ptr<kp::Tensor>& inA, const std::shared_ptr<kp::Tensor>& inA,
const std::shared_ptr<kp::Tensor>& inB, const std::shared_ptr<kp::Tensor>& inB,
const std::shared_ptr<kp::Tensor>& out, const std::shared_ptr<kp::Tensor>& out,
uint32_t inAOff, uint32_t inBOff, uint32_t outOff, uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
int32_t ne00, int32_t ne10, int32_t ne0, int32_t ne1, int32_t ne00, int32_t ne01, int32_t ne02,
int32_t ne01, int32_t ne11, int32_t ne12, int32_t ne02 int32_t ne10, int32_t ne11, int32_t ne12, int32_t ne13,
int32_t ne0, int32_t ne1,
uint32_t r2, uint32_t r3
) { ) {
struct PushConstants { struct PushConstants {
uint32_t inAOff, inBOff, outOff; uint32_t inAOff, inBOff, outOff;
int32_t ne00, ne10, ne0, ne1, ne01, gqa; int32_t ne00, ne01, ne02;
int32_t ne10, ne12;
int32_t ne0, ne1;
uint32_t r2, r3;
} pushConsts { } pushConsts {
safe_divide(inAOff, block_size), safe_divide(inBOff, 4), safe_divide(outOff, 4), safe_divide(inAOff, block_size), safe_divide(inBOff, 4), safe_divide(outOff, 4),
ne00, ne10, ne0, ne1, ne01, ne12/ne02 ne00, ne01, ne02,
ne10, ne12,
ne0, ne1,
r2, r3
}; };
auto name = std::string(__func__) + "_" + suffix; auto name = std::string(__func__) + "_" + suffix;
std::shared_ptr<kp::Algorithm> s_algo = nullptr; std::shared_ptr<kp::Algorithm> s_algo = nullptr;
if (!komputeManager()->hasAlgorithm(name)) { if (!komputeManager()->hasAlgorithm(name)) {
const uint32_t local_x = ggml_vk_current_device().subgroupSize * 2; const uint32_t local_x = ggml_vk_current_device().subgroupSize * 2;
s_algo = komputeManager()->algorithm<uint32_t, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {unsigned((ne01 + 7)/8), unsigned(ne11), unsigned(ne12)}, {local_x}, {pushConsts}); s_algo = komputeManager()->algorithm<uint32_t, PushConstants>(__func__, s_kompute_context->pool.get(), {inA, inB, out}, spirv, {unsigned((ne01 + 7)/8), unsigned(ne11), unsigned(ne12*ne13)}, {local_x}, {pushConsts});
} else { } else {
s_algo = komputeManager()->getAlgorithm(name); s_algo = komputeManager()->getAlgorithm(name);
s_algo->setTensors({inA, inB, out}); s_algo->setTensors({inA, inB, out});
s_algo->setWorkgroup({unsigned((ne01 + 7)/8), unsigned(ne11), unsigned(ne12)}); s_algo->setWorkgroup({unsigned((ne01 + 7)/8), unsigned(ne11), unsigned(ne12*ne13)});
s_algo->setPushConstants<PushConstants>({pushConsts}); s_algo->setPushConstants<PushConstants>({pushConsts});
s_algo->updateDescriptors(s_kompute_context->pool.get()); s_algo->updateDescriptors(s_kompute_context->pool.get());
} }
@ -1040,7 +1048,7 @@ static void ggml_vk_mul_mat_q4_0(Args&&... args) {
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q4_0_comp_spv, const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q4_0_comp_spv,
kp::shader_data::op_mul_mat_q4_0_comp_spv_len); kp::shader_data::op_mul_mat_q4_0_comp_spv_len);
ggml_vk_mul_mat_q4_x(spirv, "q4_0", 1/*We access blocks unaligned*/, std::forward<Args>(args)...); ggml_vk_mul_mat_impl(spirv, "q4_0", 1/*We access blocks unaligned*/, std::forward<Args>(args)...);
} }
template <typename... Args> template <typename... Args>
@ -1048,16 +1056,18 @@ static void ggml_vk_mul_mat_q4_1(Args&&... args) {
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q4_1_comp_spv, const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q4_1_comp_spv,
kp::shader_data::op_mul_mat_q4_1_comp_spv_len); kp::shader_data::op_mul_mat_q4_1_comp_spv_len);
ggml_vk_mul_mat_q4_x(spirv, "q4_1", 1/*We access blocks unaligned*/, std::forward<Args>(args)...); ggml_vk_mul_mat_impl(spirv, "q4_1", 1/*We access blocks unaligned*/, std::forward<Args>(args)...);
} }
static void ggml_vk_mul_mat_q6_k(kp::Sequence& seq, static void ggml_vk_mul_mat_q6_k(
const std::shared_ptr<kp::Tensor>& inA, kp::Sequence& seq,
const std::shared_ptr<kp::Tensor>& inB, const std::shared_ptr<kp::Tensor>& inA,
const std::shared_ptr<kp::Tensor>& out, const std::shared_ptr<kp::Tensor>& inB,
uint32_t inAOff, uint32_t inBOff, uint32_t outOff, const std::shared_ptr<kp::Tensor>& out,
int32_t ne00, int32_t ne10, int32_t ne0, int32_t ne1, uint32_t inAOff, uint32_t inBOff, uint32_t outOff,
int32_t ne01, int32_t ne11, int32_t ne12, int32_t ne02) { int32_t ne00, int32_t ne10, int32_t ne0, int32_t ne1,
int32_t ne01, int32_t ne11, int32_t ne12, int32_t ne02
) {
const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q6_k_comp_spv, const static auto spirv = getSpirvShader(kp::shader_data::op_mul_mat_q6_k_comp_spv,
kp::shader_data::op_mul_mat_q6_k_comp_spv_len); kp::shader_data::op_mul_mat_q6_k_comp_spv_len);
@ -1550,6 +1560,15 @@ void ggml_vk_graph_compute(struct ggml_kompute_context * ctx, struct ggml_cgraph
} break; } break;
case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT:
{ {
GGML_ASSERT(ne00 == ne10);
// TODO: assert that dim2 and dim3 are contiguous
GGML_ASSERT(ne12 % ne02 == 0);
GGML_ASSERT(ne13 % ne03 == 0);
const uint32_t r2 = ne12/ne02;
const uint32_t r3 = ne13/ne03;
if (src1t != GGML_TYPE_F32) { if (src1t != GGML_TYPE_F32) {
fprintf(stderr, "%s: %s: Unsupported src1 type: %u/%u\n", __func__, ggml_op_name(dst->op), src0t, src1t); fprintf(stderr, "%s: %s: Unsupported src1 type: %u/%u\n", __func__, ggml_op_name(dst->op), src0t, src1t);
goto not_implemented; goto not_implemented;
@ -1563,29 +1582,40 @@ void ggml_vk_graph_compute(struct ggml_kompute_context * ctx, struct ggml_cgraph
switch (src0t) { switch (src0t) {
case GGML_TYPE_F32: case GGML_TYPE_F32:
ggml_vk_mul_mat_mat_f32(seq, ggml_vk_mul_mat_mat_f32(
id_src0, id_src1, id_dst, seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
off_src0, off_src1, off_dst, ne00, ne01, ne02, nb01, nb02, ne11, ne12, nb11, nb12, nb1, nb2
ne00, ne01, ne02, );
nb01, nb02,
ne11, ne12,
nb11, nb12,
nb1, nb2);
break; break;
case GGML_TYPE_F16: case GGML_TYPE_F16:
ggml_vk_mul_mat_f16(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, ne01, ne02, nb01, nb02, ne11, ne12, nb11, nb12, ne0, ne1); ggml_vk_mul_mat_f16(
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
ne00, ne01, ne02, nb01, nb02, ne11, ne12, nb11, nb12, ne0, ne1
);
break; break;
case GGML_TYPE_Q8_0: case GGML_TYPE_Q8_0:
ggml_vk_mul_mat_q8_0(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, ne01, nb01, nb02, ne11, ne12, nb11, nb12, ne0, ne1); ggml_vk_mul_mat_q8_0(
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
ne00, ne01, nb01, nb02, ne11, ne12, nb11, nb12, ne0, ne1
);
break; break;
case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_0:
ggml_vk_mul_mat_q4_0(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, ne10, ne0, ne1, ne01, ne11, ne12, ne02); ggml_vk_mul_mat_q4_0(
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
ne00, ne01, ne02, ne10, ne11, ne12, ne13, ne0, ne1, r2, r3
);
break; break;
case GGML_TYPE_Q4_1: case GGML_TYPE_Q4_1:
ggml_vk_mul_mat_q4_1(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, ne10, ne0, ne1, ne01, ne11, ne12, ne02); ggml_vk_mul_mat_q4_1(
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
ne00, ne01, ne02, ne10, ne11, ne12, ne13, ne0, ne1, r2, r3
);
break; break;
case GGML_TYPE_Q6_K: case GGML_TYPE_Q6_K:
ggml_vk_mul_mat_q6_k(seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst, ne00, ne10, ne0, ne1, ne01, ne11, ne12, ne02); ggml_vk_mul_mat_q6_k(
seq, id_src0, id_src1, id_dst, off_src0, off_src1, off_dst,
ne00, ne10, ne0, ne1, ne01, ne11, ne12, ne02
);
break; break;
default: { default: {
fprintf(stderr, "%s: %s: Unsupported quantization: %u/%u\n", __func__, ggml_op_name(dst->op), src0t, src1t); fprintf(stderr, "%s: %s: Unsupported quantization: %u/%u\n", __func__, ggml_op_name(dst->op), src0t, src1t);

20
kompute-shaders/op_mul_mat_q4_0.comp
View file

@ -6,25 +6,7 @@
#define SIZE_OF_BLOCK sizeof_block_q4_0 #define SIZE_OF_BLOCK sizeof_block_q4_0
#define N_ROWS 4 #define N_ROWS 4
layout(local_size_x_id = 0) in; #include "op_mul_mv_q_n_pre.comp"
layout(local_size_y = 1) in;
layout(local_size_z = 1) in;
layout (binding = 0) readonly buffer tensorInA { uint8_t inA[]; };
layout (binding = 1) readonly buffer tensorInB { float inB[]; };
layout (binding = 2) writeonly buffer tensorOut { float out_[]; };
layout (push_constant) uniform parameter {
uint inAOff;
uint inBOff;
uint outOff;
int ne00;
int ne10;
int ne0;
int ne1;
int ne01;
int gqa;
} pcs;
// The q4_0 version of this function // The q4_0 version of this function
float block_q_n_dot_y(uint block_index, uint yb, uint il) { float block_q_n_dot_y(uint block_index, uint yb, uint il) {

20
kompute-shaders/op_mul_mat_q4_1.comp
View file

@ -6,25 +6,7 @@
#define SIZE_OF_BLOCK sizeof_block_q4_1 #define SIZE_OF_BLOCK sizeof_block_q4_1
#define N_ROWS 4 #define N_ROWS 4
layout(local_size_x_id = 0) in; #include "op_mul_mv_q_n_pre.comp"
layout(local_size_y = 1) in;
layout(local_size_z = 1) in;
layout (binding = 0) readonly buffer tensorInA { uint8_t inA[]; };
layout (binding = 1) readonly buffer tensorInB { float inB[]; };
layout (binding = 2) writeonly buffer tensorOut { float out_[]; };
layout (push_constant) uniform parameter {
uint inAOff;
uint inBOff;
uint outOff;
int ne00;
int ne10;
int ne0;
int ne1;
int ne01;
int gqa;
} pcs;
// The q4_1 version of this function // The q4_1 version of this function
float block_q_n_dot_y(uint block_index, uint yb, uint il) { float block_q_n_dot_y(uint block_index, uint yb, uint il) {

9
kompute-shaders/op_mul_mv_q_n.comp
View file

@ -1,13 +1,20 @@
void main() { void main() {
// NB: hack to make compatible with AMD GPUs that have a subgroup size of 64
if (gl_SubgroupInvocationID > 31) if (gl_SubgroupInvocationID > 31)
return; return;
const uint nb = uint(pcs.ne00/BLOCKS_IN_QUANT); const uint nb = uint(pcs.ne00/BLOCKS_IN_QUANT);
const uint r0 = gl_WorkGroupID.x; const uint r0 = gl_WorkGroupID.x;
const uint r1 = gl_WorkGroupID.y; const uint r1 = gl_WorkGroupID.y;
const uint im = gl_WorkGroupID.z; const uint im = gl_WorkGroupID.z;
const uint first_row = (r0 * gl_NumSubgroups + gl_SubgroupID) * N_ROWS; const uint first_row = (r0 * gl_NumSubgroups + gl_SubgroupID) * N_ROWS;
const uint offset0 = first_row * nb + im/pcs.gqa*(nb*pcs.ne0);
const uint i12 = im%pcs.ne12;
const uint i13 = im/pcs.ne12;
const uint offset0 = first_row * nb + (i12/pcs.r2)*(nb*pcs.ne01) + (i13/pcs.r3)*(nb*pcs.ne01*pcs.ne02);
const uint x = offset0; // Based from inA without base offset const uint x = offset0; // Based from inA without base offset
const uint y = r1*uint(pcs.ne10)+im*pcs.ne00*pcs.ne1+pcs.inBOff; // Based from inB const uint y = r1*uint(pcs.ne10)+im*pcs.ne00*pcs.ne1+pcs.inBOff; // Based from inB

22
kompute-shaders/op_mul_mv_q_n_pre.comp Normal file
View file

@ -0,0 +1,22 @@
layout(local_size_x_id = 0) in;
layout(local_size_y = 1) in;
layout(local_size_z = 1) in;
layout (binding = 0) readonly buffer tensorInA { uint8_t inA[]; };
layout (binding = 1) readonly buffer tensorInB { float inB[]; };
layout (binding = 2) writeonly buffer tensorOut { float out_[]; };
layout (push_constant) uniform parameter {
uint inAOff;
uint inBOff;
uint outOff;
int ne00;
int ne01;
int ne02;
int ne10;
int ne12;
int ne0;
int ne1;
uint r2;
uint r3;
} pcs;