vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cuda : fix multi-gpu with tensor cores

Browse source
This commit is contained in:
Georgi Gerganov 2023-10-27 23:11:50 +03:00
parent 706ff4c2e0
commit cd3e20fb50
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
ggml-cuda.cu
View file

@ -461,7 +461,12 @@ static_assert(K_QUANTS_PER_ITERATION == 1 || K_QUANTS_PER_ITERATION == 2, "K_QUA
#define GGML_CUDA_PEER_MAX_BATCH_SIZE 128
#endif // GGML_CUDA_PEER_MAX_BATCH_SIZE
#ifdef GGML_CUDA_FORCE_MMQ
#define MUL_MAT_SRC1_COL_STRIDE 128
#else
// with tensor cores, we copy the entire hidden state to the devices in one go
#define MUL_MAT_SRC1_COL_STRIDE 4096
#endif
#define MAX_STREAMS 8
static cudaStream_t g_cudaStreams[GGML_CUDA_MAX_DEVICES][MAX_STREAMS] = { nullptr };