vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

revert it

Browse source
This commit is contained in:
Eve 2025-01-04 14:57:40 -05:00
parent 21c6b805c9
commit b0e4ccbeb9
4 changed files with 34 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

16
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp
View file

@ -14,13 +14,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
// 16 threads are used to process each block // 16 threads are used to process each block
const uint it_size = gl_WorkGroupSize.x/16; const uint it_size = gl_WorkGroupSize.x/16;
const uint tid = gl_LocalInvocationID.x; const uint tid = gl_LocalInvocationID.x;
const uint itid = tid%16; // 0...16 const uint itid = tid%16; // 0...15
const uint ix = tid/16; const uint ix = tid/16;
const uint step = 8; const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
const uint v_in = itid - 8*v_im; // 0...15 or 0...7
const uint v_im = itid/step; // 0 or 1. 0 computes 0..., 1 computes 128...
const uint v_in = itid - step*v_im; // 0...15 or 0...7
const uint l0 = 2*v_in; // 0...15 const uint l0 = 2*v_in; // 0...15
const uint q_offset = 32*v_im + l0; const uint q_offset = 32*v_im + l0;
@ -44,7 +42,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
const FLOAT_TYPE dall = d.x; const FLOAT_TYPE dall = d.x;
const FLOAT_TYPE dmin = d.y; const FLOAT_TYPE dmin = d.y;
uint32_t s0_u32 = data_a_packed32[ib0 + i].scales[s_offset / 4 + 0]; uint32_t s0_u32 = data_a_packed32[ib0 + i].scales[s_offset / 4];
uint32_t s4_u32 = data_a_packed32[ib0 + i].scales[s_offset / 4 + 1]; uint32_t s4_u32 = data_a_packed32[ib0 + i].scales[s_offset / 4 + 1];
uint32_t s0_lo4_u32 = s0_u32 & 0x0F0F0F0F; uint32_t s0_lo4_u32 = s0_u32 & 0x0F0F0F0F;
@ -57,13 +55,13 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
uvec4 s0_hi4 = uvec4(unpack8(s0_hi4_u32)); uvec4 s0_hi4 = uvec4(unpack8(s0_hi4_u32));
uvec4 s4_hi4 = uvec4(unpack8(s4_hi4_u32)); uvec4 s4_hi4 = uvec4(unpack8(s4_hi4_u32));
uint16_t qs0_u16 = data_a_packed16[ib0 + i].qs[q_offset / 2 + 0]; uint16_t qs0_u16 = data_a_packed16[ib0 + i].qs[q_offset / 2];
uint16_t qs16_u16 = data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]; uint16_t qs16_u16 = data_a_packed16[ib0 + i].qs[q_offset / 2 + 8];
uvec2 qs0 = uvec2(unpack8(qs0_u16)); uvec2 qs0 = uvec2(unpack8(qs0_u16));
uvec2 qs16 = uvec2(unpack8(qs16_u16)); uvec2 qs16 = uvec2(unpack8(qs16_u16));
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]; B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2];
B_TYPE_VEC2 b16 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]; B_TYPE_VEC2 b16 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8];
B_TYPE_VEC2 b32 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]; B_TYPE_VEC2 b32 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16];
B_TYPE_VEC2 b48 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]; B_TYPE_VEC2 b48 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24];

12
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp
View file

@ -14,13 +14,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
// 16 threads are used to process each block // 16 threads are used to process each block
const uint it_size = gl_WorkGroupSize.x/16; const uint it_size = gl_WorkGroupSize.x/16;
const uint tid = gl_LocalInvocationID.x; const uint tid = gl_LocalInvocationID.x;
const uint itid = tid%16; // 0...16 const uint itid = tid%16; // 0...15
const uint ix = tid/16; const uint ix = tid/16;
const uint step = 8; const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
const uint v_in = itid - 8*v_im; // 0...15 or 0...7
const uint v_im = itid/step; // 0 or 1. 0 computes 0..., 1 computes 128...
const uint v_in = itid - step*v_im; // 0...15 or 0...7
const uint8_t m = uint8_t(1 << (4 * v_im)); const uint8_t m = uint8_t(1 << (4 * v_im));
@ -60,7 +58,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]; B_TYPE_VEC2 b0 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2];
B_TYPE_VEC2 b16 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]; B_TYPE_VEC2 b16 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8];
B_TYPE_VEC2 b32 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]; B_TYPE_VEC2 b32 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16];
B_TYPE_VEC2 b48 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]; B_TYPE_VEC2 b48 = data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24];

49
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp
View file

@ -6,8 +6,6 @@
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
shared FLOAT_TYPE sc[BLOCK_SIZE/16][2][8];
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
uint a_offset, b_offset, d_offset; uint a_offset, b_offset, d_offset;
get_offsets(a_offset, b_offset, d_offset); get_offsets(a_offset, b_offset, d_offset);
@ -27,16 +25,6 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224 const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224
const uint v_in = il % 2; const uint v_in = il % 2;
// v_im may be incorrect
const uint itid8 = itid%8;
const uint a = (itid8 > 5) ? 4 : 0;
const uint b = (itid8 > 3) ? 0x0f : 0x3f;
const uint c = (itid8 > 3) ? 0xc0 : 0xFF;
const uint dd = (itid8 > 3) ? 2 : 0;
const uint gg = (itid8 > 1) ? 1 : 0;
const uint g = (itid8 > 3) ? 8 : gg;
const uint h = itid8%2;
const uint l0 = n * (2 * ir + v_in); // 0...15 const uint l0 = n * (2 * ir + v_in); // 0...15
const uint q_offset = 32*v_im + l0; const uint q_offset = 32*v_im + l0;
const uint y_offset = 64*v_im + l0; const uint y_offset = 64*v_im + l0;
@ -62,23 +50,18 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ]; const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ];
const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2]; const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2];
const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4]; const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4];
uvec2 scale0 = uvec2(unpack8(scale0_u32));
uvec2 scale4 = uvec2(unpack8(scale4_u32));
uvec2 scale8 = uvec2(unpack8(scale8_u32));
uvec2 scale[3]; const uint32_t sc0 = ( scale0.x & 0x3f);
scale[0] = uvec2(unpack8(scale0_u32)); const uint32_t sc1 = ( scale0.y & 0x3f);
scale[1] = uvec2(unpack8(scale4_u32)); const uint32_t sc2 = ( scale4.x & 0x3f);
scale[2] = uvec2(unpack8(scale8_u32)); const uint32_t sc3 = ( scale4.y & 0x3f);
const uint32_t sc4 = (( scale8.x & 0x0f) | ((scale0.x & 0xc0) >> 2));
sc[ix][v_im][itid8] = FLOAT_TYPE(((scale[gg][h] >> a) & b) | ((scale[g][h] & c) >> dd)); const uint32_t sc5 = (( scale8.y & 0x0f) | ((scale0.y & 0xc0) >> 2));
barrier(); const uint32_t sc6 = (((scale8.x >> 4) & 0x0f) | ((scale4.x & 0xc0) >> 2));
const uint32_t sc7 = (((scale8.y >> 4) & 0x0f) | ((scale4.y & 0xc0) >> 2));
//const uint32_t sc0 = ( scale0.x & 0x3f);
//const uint32_t sc1 = ( scale0.y & 0x3f);
//const uint32_t sc2 = ( scale4.x & 0x3f);
//const uint32_t sc3 = ( scale4.y & 0x3f);
//const uint32_t sc4 = (( scale8.x & 0x0f) | ((scale0.x & 0xc0) >> 2));
//const uint32_t sc5 = (( scale8.y & 0x0f) | ((scale0.y & 0xc0) >> 2));
//const uint32_t sc6 = (((scale8.x >> 4) & 0x0f) | ((scale4.x & 0xc0) >> 2));
//const uint32_t sc7 = (((scale8.y >> 4) & 0x0f) | ((scale4.y & 0xc0) >> 2));
uint32_t qs0_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4]; uint32_t qs0_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4];
uint32_t qs64_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4 + 16]; uint32_t qs64_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4 + 16];
@ -121,11 +104,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
const FLOAT_TYPE sz = fma(FLOAT_TYPE(by20.x), q4_8, fma(FLOAT_TYPE(by20.y), q4_9, fma(FLOAT_TYPE(by20.z), q4_10, FLOAT_TYPE(by20.w) * q4_11))); const FLOAT_TYPE sz = fma(FLOAT_TYPE(by20.x), q4_8, fma(FLOAT_TYPE(by20.y), q4_9, fma(FLOAT_TYPE(by20.z), q4_10, FLOAT_TYPE(by20.w) * q4_11)));
const FLOAT_TYPE sw = fma(FLOAT_TYPE(by232.x), q4_12, fma(FLOAT_TYPE(by232.y), q4_13, fma(FLOAT_TYPE(by232.z), q4_14, FLOAT_TYPE(by232.w) * q4_15))); const FLOAT_TYPE sw = fma(FLOAT_TYPE(by232.x), q4_12, fma(FLOAT_TYPE(by232.y), q4_13, fma(FLOAT_TYPE(by232.z), q4_14, FLOAT_TYPE(by232.w) * q4_15)));
const FLOAT_TYPE smin = const FLOAT_TYPE smin =
fma(FLOAT_TYPE(by10.x), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.x), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.x), sc[ix][v_im][6], fma(FLOAT_TYPE(by232.x), sc[ix][v_im][7], fma(FLOAT_TYPE(by10.x), sc2, fma(FLOAT_TYPE(by132.x), sc3, fma(FLOAT_TYPE(by20.x), sc6, fma(FLOAT_TYPE(by232.x), sc7,
fma(FLOAT_TYPE(by10.y), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.y), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.y), sc[ix][v_im][6], fma(FLOAT_TYPE(by232.y), sc[ix][v_im][7], fma(FLOAT_TYPE(by10.y), sc2, fma(FLOAT_TYPE(by132.y), sc3, fma(FLOAT_TYPE(by20.y), sc6, fma(FLOAT_TYPE(by232.y), sc7,
fma(FLOAT_TYPE(by10.z), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.z), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.z), sc[ix][v_im][6], fma(FLOAT_TYPE(by232.z), sc[ix][v_im][7], fma(FLOAT_TYPE(by10.z), sc2, fma(FLOAT_TYPE(by132.z), sc3, fma(FLOAT_TYPE(by20.z), sc6, fma(FLOAT_TYPE(by232.z), sc7,
fma(FLOAT_TYPE(by10.w), sc[ix][v_im][2], fma(FLOAT_TYPE(by132.w), sc[ix][v_im][3], fma(FLOAT_TYPE(by20.w), sc[ix][v_im][6], FLOAT_TYPE(by232.w) * sc[ix][v_im][7]))))))))))))))); fma(FLOAT_TYPE(by10.w), sc2, fma(FLOAT_TYPE(by132.w), sc3, fma(FLOAT_TYPE(by20.w), sc6, FLOAT_TYPE(by232.w) * sc7)))))))))))))));
temp[j][n] = fma(dall, fma(sx, sc[ix][v_im][0], fma(sy, sc[ix][v_im][1], fma(sz, sc[ix][v_im][4], sw * sc[ix][v_im][5]))), fma(-dmin, smin, temp[j][n])); temp[j][n] = fma(dall, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dmin, smin, temp[j][n]));
} }
} }
} }

12
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp
View file

@ -15,8 +15,8 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
// 16 threads are used to process each block // 16 threads are used to process each block
const uint it_size = gl_WorkGroupSize.x/16; const uint it_size = gl_WorkGroupSize.x/16;
const uint tid = gl_LocalInvocationID.x; const uint tid = gl_LocalInvocationID.x;
const uint itid = tid%16; // 0...16 const uint itid = tid%16; // 0...15
const uint ix = tid/16; const uint ix = tid/16;
const uint il = itid/4; // 0...3 const uint il = itid/4; // 0...3
const uint ir = itid - 4*il; // 0...7 or 0...3 const uint ir = itid - 4*il; // 0...7 or 0...3
@ -49,9 +49,9 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ]; uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ];
uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2]; uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2];
uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4]; uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4];
uvec4 scale0 = uvec4(unpack8(scale0_u32)); uvec2 scale0 = uvec2(unpack8(scale0_u32));
uvec4 scale4 = uvec4(unpack8(scale4_u32)); uvec2 scale4 = uvec2(unpack8(scale4_u32));
uvec4 scale8 = uvec4(unpack8(scale8_u32)); uvec2 scale8 = uvec2(unpack8(scale8_u32));
const uint32_t sc0 = ( scale0.x & 0x3f); const uint32_t sc0 = ( scale0.x & 0x3f);
const uint32_t sc1 = ( scale0.y & 0x3f); const uint32_t sc1 = ( scale0.y & 0x3f);
@ -74,7 +74,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
uint32_t qs0_16_lo4_offset16 = ((qh >> (2*v_im)) & 0x01010101) << 4; uint32_t qs0_16_lo4_offset16 = ((qh >> (2*v_im)) & 0x01010101) << 4;
uint32_t qs0_16_hi4_offset16 = ((qh >> (2*v_im)) & 0x02020202) << 3; uint32_t qs0_16_hi4_offset16 = ((qh >> (2*v_im)) & 0x02020202) << 3;
uint32_t qs64_80_lo4_offset16 = ((qh >> (2*v_im)) & 0x10101010) << 0; uint32_t qs64_80_lo4_offset16 = ((qh >> (2*v_im)) & 0x10101010);
uint32_t qs64_80_hi4_offset16 = ((qh >> (2*v_im)) & 0x20202020) >> 1; uint32_t qs64_80_hi4_offset16 = ((qh >> (2*v_im)) & 0x20202020) >> 1;
qs0_16_u32_lo4 += qs0_16_lo4_offset16; qs0_16_u32_lo4 += qs0_16_lo4_offset16;