vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update Vulkan CPU offload for MUL_MAT_ID

Browse source
This commit is contained in:
0cc4m 2024-06-02 09:05:47 +02:00
parent 2c3d0b42f3
commit 6e0e0beb56
1 changed files with 2 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

9
ggml-vulkan.cpp
View file

@ -6516,15 +6516,10 @@ GGML_CALL static bool ggml_backend_vk_supports_op(ggml_backend_t backend, const
}
GGML_CALL static bool ggml_backend_vk_offload_op(ggml_backend_t backend, const ggml_tensor * op) {
const ggml_tensor * dst = op;
const int min_batch_size = 32;
if (dst->ne[1] > min_batch_size && dst->op != GGML_OP_GET_ROWS) {
return true;
}
return false;
return (op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS) ||
(op->ne[2] >= min_batch_size && op->op == GGML_OP_MUL_MAT_ID);
UNUSED(backend);
}