vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

kompute : only try to use Vulkan for LLaMA itself

Browse source
This commit is contained in:
Cebtenzzre 2023-10-04 16:16:04 -04:00 • committed by cebtenzzre
parent bc4b5ed1cb
commit 24a4a5956a
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama.cpp
View file

@ -6492,7 +6492,9 @@ struct llama_context * llama_new_context_with_model(
#undef LLAMA_METAL_CHECK_BUF
}
#elif defined(GGML_USE_KOMPUTE)
// TODO(cebtenzzre): we need to check the type of each tensor because Q8_0 is not currently supported
if (ggml_vk_has_device() && params.n_gpu_layers > 0
&& model->arch == LLM_ARCH_LLAMA
&& (model->ftype == LLAMA_FTYPE_ALL_F32
|| model->ftype == LLAMA_FTYPE_MOSTLY_F16
|| model->ftype == LLAMA_FTYPE_MOSTLY_Q4_0