use stride=128 if built for tensor cores
This commit is contained in:
parent
dd71a35cc8
commit
6272b6764a
1 changed files with 6 additions and 1 deletions
|
@ -468,7 +468,12 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA
|
|||
#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
|
||||
|
||||
#define MUL_MAT_SRC1_COL_STRIDE_MMQ 128
|
||||
#define MUL_MAT_SRC1_COL_STRIDE 4096
|
||||
|
||||
#ifdef CUDA_USE_TENSOR_CORES
|
||||
#define MUL_MAT_SRC1_COL_STRIDE 128
|
||||
#else
|
||||
#define MUL_MAT_SRC1_COL_STRIDE 4096
|
||||
#endif
|
||||
|
||||
#define MAX_STREAMS 8
|
||||
static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { { nullptr } };
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue