vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Account for GGUF_KEY_KEY only setting when the key exists

Browse source
This commit is contained in:
KerfuffleV2 2023-10-15 09:48:30 -06:00
parent 32383bbd1c
commit 4079668cda
1 changed files with 13 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

26
llama.cpp
View file

@ -2242,23 +2242,23 @@ static void llm_load_vocab(
// special tokens
{
const std::vector<std::tuple<enum llm_kv, int32_t *>> special_token_types = {
{ LLM_KV_TOKENIZER_BOS_ID, &vocab.special_bos_id },
{ LLM_KV_TOKENIZER_EOS_ID, &vocab.special_eos_id },
{ LLM_KV_TOKENIZER_UNK_ID, &vocab.special_unk_id },
{ LLM_KV_TOKENIZER_SEP_ID, &vocab.special_sep_id },
{ LLM_KV_TOKENIZER_PAD_ID, &vocab.special_pad_id },
const std::vector<std::tuple<enum llm_kv, int32_t &>> special_token_types = {
{ LLM_KV_TOKENIZER_BOS_ID, vocab.special_bos_id },
{ LLM_KV_TOKENIZER_EOS_ID, vocab.special_eos_id },
{ LLM_KV_TOKENIZER_UNK_ID, vocab.special_unk_id },
{ LLM_KV_TOKENIZER_SEP_ID, vocab.special_sep_id },
{ LLM_KV_TOKENIZER_PAD_ID, vocab.special_pad_id },
};
for (auto & it : special_token_types ) {
int32_t id = -1;
const std::string kstr = kv(std::get<0>(it));
for (const auto & it : special_token_types ) {
const std::string key = kv(std::get<0>(it));
int32_t & id = std::get<1>(it), old_id = id;
GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, kstr);
GGUF_GET_KEY(ctx, id, gguf_get_val_u32, GGUF_TYPE_UINT32, false, key);
if (id != -1 && (id < 0 || size_t(id) >= vocab.id_to_token.size())) {
LLAMA_LOG_WARN("%s: bad special token value %d for key '%s' -- ignoring\n", __func__, id, kstr.c_str());
continue;
LLAMA_LOG_WARN("%s: bad special token: '%s' = %d, using default id %d\n",
__func__, key.c_str(), id, old_id);
id = old_id;
}
*(std::get<1>(it)) = id;
}
}