vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

examplse : de-shadow

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2025-01-12 14:25:32 +02:00
parent 82caffa74e
commit 9a735ae6d8
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
16 changed files with 152 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

8
examples/server/utils.hpp
View file

@ -129,15 +129,15 @@ static llama_tokens tokenize_mixed(const llama_vocab * vocab, const json & json_
if (p.is_string()) {
auto s = p.template get<std::string>();
llama_tokens p;
llama_tokens ids;
if (first) {
p = common_tokenize(vocab, s, add_special, parse_special);
ids = common_tokenize(vocab, s, add_special, parse_special);
first = false;
} else {
p = common_tokenize(vocab, s, false, parse_special);
ids = common_tokenize(vocab, s, false, parse_special);
}
prompt_tokens.insert(prompt_tokens.end(), p.begin(), p.end());
prompt_tokens.insert(prompt_tokens.end(), ids.begin(), ids.end());
} else {
if (first) {
first = false;