vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : print more info about control tokens

Browse source
This commit is contained in:
Georgi Gerganov 2024-10-13 13:42:16 +03:00
parent 491f211b4c
commit 4f46e29b09
No known key found for this signature in database
GPG key ID: BF970631944C16B7
1 changed files with 0 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/llama.cpp
View file

@ -6725,10 +6725,6 @@ static void llm_load_vocab(
vocab.special_eog_ids.insert(vocab.special_eom_id);
LLAMA_LOG_WARN("%s: special_eom_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
}
if (vocab.special_fim_sep_id != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(vocab.special_fim_sep_id) == 0) {
vocab.special_eog_ids.insert(vocab.special_fim_sep_id);
}
}
// build special tokens cache