vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add logit bias support

Browse source
This commit is contained in:
Henri Vasserman 2023-06-01 13:57:47 +03:00
parent 8c6a5fc92b
commit 9531ae60db
No known key found for this signature in database
GPG key ID: 2995FC0F58B1A986
1 changed files with 10 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

10
examples/server/server.cpp
View file

@ -655,6 +655,16 @@ bool parse_options_completion(json body, llama_server_context& llama, Response &
} else { } else {
llama.params.logit_bias.erase(llama_token_eos()); llama.params.logit_bias.erase(llama_token_eos());
} }
if (body["logit_bias"].is_array()) {
int n_vocab = llama_n_vocab(llama.ctx);
for (const auto &el : body["logit_bias"]) {
if (el.is_array() && el.size() == 2 && el[0].is_number_integer() && el[1].is_number_float()) {
llama_token tok = el[0].get<llama_token>();
if (tok < 0 || tok >= n_vocab) continue;
llama.params.logit_bias[tok] = el[1].get<float>();
}
}
}
if (!body["prompt"].is_null()) { if (!body["prompt"].is_null()) {
llama.params.prompt = body["prompt"].get<std::string>(); llama.params.prompt = body["prompt"].get<std::string>();
} else { } else {