CUDA: mmq CLI option, fixed mmq build issues (#2453)
This commit is contained in:
parent
1215ed7d5c
commit
0728c5a8b9
10 changed files with 67 additions and 27 deletions
1
llama.h
1
llama.h
|
@ -108,6 +108,7 @@ extern "C" {
|
|||
|
||||
// Keep the booleans together to avoid misalignment during copy-by-value.
|
||||
bool low_vram; // if true, reduce VRAM usage at the cost of performance
|
||||
bool mul_mat_q; // if true, use experimental mul_mat_q kernels
|
||||
bool f16_kv; // use fp16 for KV cache
|
||||
bool logits_all; // the llama_eval() call computes all logits, not just the last one
|
||||
bool vocab_only; // only load the vocabulary, no weights
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue