Fix BPE newline check, only I could break something so simple
This commit is contained in:
parent
d1075f6e08
commit
14be9d9141
1 changed files with 1 additions and 1 deletions
|
@ -2236,7 +2236,7 @@ static void llm_load_vocab(
|
||||||
vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
|
vocab.linefeed_id = llama_byte_to_token(vocab, '\n');
|
||||||
} else {
|
} else {
|
||||||
const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false);
|
const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false);
|
||||||
GGML_ASSERT(ids.size() == 1 && "model vocab missing newline token");
|
GGML_ASSERT(ids.empty() && "model vocab missing newline token");
|
||||||
vocab.linefeed_id = ids[0];
|
vocab.linefeed_id = ids[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue