Force measure to allocate more memory for 70Bs
This commit is contained in:
parent
fae6d9c70d
commit
d6b44fb3ae
1 changed files with 5 additions and 8 deletions
13
llama.cpp
13
llama.cpp
|
@ -3266,14 +3266,11 @@ static struct ggml_cgraph * llm_build_llama(
|
|||
} else {
|
||||
run_layer = NULL;
|
||||
}
|
||||
} else if (ggml_allocr_is_measure(lctx.alloc) && il == n_layer - 1) {
|
||||
// No idea why this is needed, but otherwise we run out of space
|
||||
// when skipping attn or mlp (but not both) on the last layer
|
||||
run_mlp = false;
|
||||
} else if (ggml_allocr_is_measure(lctx.alloc) && il == n_layer - 2) {
|
||||
// No idea why this is needed, but otherwise we run out of space
|
||||
// when skipping attn or mlp (but not both) on the last layer
|
||||
run_attn = false;
|
||||
} else if (ggml_allocr_is_measure(lctx.alloc)) {
|
||||
if (il == 0 || il == n_layer - 1) run_mlp = false;
|
||||
else if (il == 1 || il == n_layer - 2) run_attn = false;
|
||||
else if (il & 1) run_mlp = false;
|
||||
else run_attn = false;
|
||||
}
|
||||
if (!run_attn && !run_mlp) continue;
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue