Compare commits

...
Sign in to create a new pull request.

2 commits

Author SHA1 Message Date
Georgi Gerganov
0cf9a06799
vocab : minor [no ci] 2025-01-14 10:36:28 +02:00
Georgi Gerganov
69fc940d9a
vocab : add dummy tokens for "no_vocab" type
ggml-ci
2025-01-14 10:28:05 +02:00

View file

@ -1356,8 +1356,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
// read vocab size from metadata // read vocab size from metadata
uint32_t n_tokens = 0; uint32_t n_tokens = 0;
if (!ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) { if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
LLAMA_LOG_WARN("%s: there is no vocab_size in metadata\n", __func__); LLAMA_LOG_WARN("%s: adding %u dummy tokens\n", __func__, n_tokens);
id_to_token.resize(n_tokens);
} }
return; return;