vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make MUL_MAT_SRC1_COL_STRIDE conditional on runtime mmq

Browse source
This commit is contained in:
Jared Van Bortel 2023-11-27 13:05:55 -05:00
parent 12fb1c58ec
commit dd71a35cc8
1 changed files with 7 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ggml-cuda.cu
View file

@ -467,6 +467,7 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA
#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128
#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
#define MUL_MAT_SRC1_COL_STRIDE_MMQ 128
#define MUL_MAT_SRC1_COL_STRIDE 4096
#define MAX_STREAMS 8
@ -7158,7 +7159,10 @@ static void ggml_cuda_op_mul_mat(
CUDA_CHECK(cudaEventRecord(src0_extra->events[g_main_device][0], g_cudaStreams[g_main_device][0]));
}
const int64_t src1_col_stride = split && used_devices > 1 ? MUL_MAT_SRC1_COL_STRIDE : ne11;
const int64_t src1_col_stride = !split || used_devices == 1 ? ne11 :
convert_src1_to_q8_1 ? MUL_MAT_SRC1_COL_STRIDE_MMQ :
MUL_MAT_SRC1_COL_STRIDE;
for (int64_t src1_col_0 = 0; src1_col_0 < ne11; src1_col_0 += src1_col_stride) {
const int64_t is = split ? (src1_col_0/src1_col_stride) % MAX_STREAMS : 0;
const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride;
@ -7296,7 +7300,7 @@ static void ggml_cuda_op_mul_mat(
// main device waits for all other devices to be finished
if (split && g_device_count > 1) {
int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE;
int64_t is_max = (ne11 + src1_col_stride - 1) / src1_col_stride;
is_max = is_max <= MAX_STREAMS ? is_max : MAX_STREAMS;
CUDA_CHECK(ggml_cuda_set_device(g_main_device));