vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix BPE newline check, only I could break something so simple

Browse source
This commit is contained in:
KerfuffleV2 2023-10-15 08:00:53 -06:00
parent d1075f6e08
commit 14be9d9141
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama.cpp
View file

@ -2236,7 +2236,7 @@ static void llm_load_vocab(
vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
} else {
const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false);
GGML_ASSERT(ids.size() == 1 && "model vocab missing newline token");
GGML_ASSERT(ids.empty() && "model vocab missing newline token");
vocab.linefeed_id = ids[0];
}