speculative: add --n-gpu-layers-draft option (#3063)
This commit is contained in:
parent
b52b29ab9d
commit
84e723653c
3 changed files with 15 additions and 0 deletions
|
@ -42,6 +42,7 @@ int main(int argc, char ** argv) {
|
|||
|
||||
// load the draft model
|
||||
params.model = params.model_draft;
|
||||
params.n_gpu_layers = params.n_gpu_layers_draft;
|
||||
std::tie(model_dft, ctx_dft) = llama_init_from_gpt_params(params);
|
||||
|
||||
// tokenize the prompt
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue