vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

common : add -hfd option for the draft model (#11318)

Browse source 
* common : add -hfd option for the draft model

* cont : fix env var

* cont : more fixes
This commit is contained in:
Georgi Gerganov 2025-01-20 22:29:43 +02:00 • committed by GitHub
parent aea8ddd516
commit 80d0d6b4b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 24 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

5
examples/server/server.cpp
View file

@ -1728,13 +1728,16 @@ struct server_context {
add_bos_token = llama_vocab_get_add_bos(vocab);
has_eos_token = llama_vocab_eos(vocab) != LLAMA_TOKEN_NULL;
if (!params_base.speculative.model.empty()) {
if (!params_base.speculative.model.empty() || !params_base.speculative.hf_repo.empty()) {
SRV_INF("loading draft model '%s'\n", params_base.speculative.model.c_str());
auto params_dft = params_base;
params_dft.devices = params_base.speculative.devices;
params_dft.hf_file = params_base.speculative.hf_file;
params_dft.hf_repo = params_base.speculative.hf_repo;
params_dft.model = params_base.speculative.model;
params_dft.model_url = params_base.speculative.model_url;
params_dft.n_ctx = params_base.speculative.n_ctx == 0 ? params_base.n_ctx / params_base.n_parallel : params_base.speculative.n_ctx;
params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers;
params_dft.n_parallel = 1;