Fix BPE newline check, only I could break something so simple

This commit is contained in:
KerfuffleV2 2023-10-15 08:00:53 -06:00
parent d1075f6e08
commit 14be9d9141

View file

@ -2236,7 +2236,7 @@ static void llm_load_vocab(
vocab.linefeed_id = llama_byte_to_token(vocab, '\n'); vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
} else { } else {
const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false); const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false);
GGML_ASSERT(ids.size() == 1 && "model vocab missing newline token"); GGML_ASSERT(ids.empty() && "model vocab missing newline token");
vocab.linefeed_id = ids[0]; vocab.linefeed_id = ids[0];
} }