speculative: add --n-gpu-layers-draft option (#3063)

This commit is contained in:
FK 2023-09-13 08:50:46 +02:00 committed by GitHub
parent b52b29ab9d
commit 84e723653c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 0 deletions

View file

@ -42,6 +42,7 @@ int main(int argc, char ** argv) {
// load the draft model
params.model = params.model_draft;
params.n_gpu_layers = params.n_gpu_layers_draft;
std::tie(model_dft, ctx_dft) = llama_init_from_gpt_params(params);
// tokenize the prompt