vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : fix norm backend

Browse source
This commit is contained in:
slaren 2024-03-13 12:18:09 +01:00
parent 4400153348
commit 9e7cecc1c8
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama.cpp
View file

@ -8367,7 +8367,9 @@ static struct ggml_cgraph * llama_build_graph(
}
}
if (il != -1) {
// norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends
// to fix this, we assign the norm layer manually to the backend of its layer
if (il != -1 && strcmp(name, "norm") == 0) {
for (auto * backend : lctx.backends) {
if (ggml_backend_buft_supports_backend(lctx.model.buft_layer[il].buft, backend)) {
ggml_backend_sched_set_tensor_backend(lctx.sched, cur, backend);