vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

vocab : add dummy tokens for "no_vocab" type

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2025-01-14 10:26:47 +02:00
parent a4f3f5d8e6
commit 69fc940d9a
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/llama-vocab.cpp
View file

@ -1356,8 +1356,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
// read vocab size from metadata
uint32_t n_tokens = 0;
if (!ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
LLAMA_LOG_WARN("%s: there is no vocab_size in metadata\n", __func__);
if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
LLAMA_LOG_WARN("%s: adding %d dummy tokens\n", __func__, n_tokens);
id_to_token.resize(n_tokens);
}
return;