vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixed vocabulary guarding assertions

Browse source
This commit is contained in:
goerch 2023-09-19 21:34:41 +02:00
parent 59a30b768a
commit a6070b7c5a
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama.cpp
View file

@ -1831,7 +1831,7 @@ static void llm_load_vocab(
for (int i = 0; i < n_merges; i++) { for (int i = 0; i < n_merges; i++) {
const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i); const std::string word = gguf_get_arr_str(ctx, merges_keyidx, i);
GGML_ASSERT(codepoints_from_utf8(word).size() >= 0); GGML_ASSERT(codepoints_from_utf8(word).size() > 0);
std::string first; std::string first;
std::string second; std::string second;
@ -1866,7 +1866,7 @@ static void llm_load_vocab(
for (uint32_t i = 0; i < n_vocab; i++) { for (uint32_t i = 0; i < n_vocab; i++) {
std::string word = gguf_get_arr_str(ctx, token_idx, i); std::string word = gguf_get_arr_str(ctx, token_idx, i);
GGML_ASSERT(codepoints_from_utf8(word).size() >= 0); GGML_ASSERT(codepoints_from_utf8(word).size() > 0);
vocab.token_to_id[word] = i; vocab.token_to_id[word] = i;