From 6272b6764ad4b39eb05a411489577d3a6c914b89 Mon Sep 17 00:00:00 2001
From: Jared Van Bortel <jared@nomic.ai>
Date: Mon, 27 Nov 2023 13:09:14 -0500
Subject: [PATCH] use stride=128 if built for tensor cores

---
 ggml-cuda.cu | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index 1bd1cbda3..51a79b19f 100644
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -468,7 +468,12 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA
 #endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
 
 #define MUL_MAT_SRC1_COL_STRIDE_MMQ 128
-#define MUL_MAT_SRC1_COL_STRIDE     4096
+
+#ifdef CUDA_USE_TENSOR_CORES
+#define MUL_MAT_SRC1_COL_STRIDE 128
+#else
+#define MUL_MAT_SRC1_COL_STRIDE 4096
+#endif
 
 #define MAX_STREAMS 8
 static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { { nullptr } };