vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : fix BPE LF token on MSVC

Browse source
This commit is contained in:
Jared Van Bortel 2024-03-23 14:03:16 -04:00
parent 6f4fd8f114
commit d05c13b3b9
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama.cpp
View file

@ -3971,7 +3971,7 @@ static void llm_load_vocab(
} else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) { } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
vocab.linefeed_id = vocab.special_pad_id; vocab.linefeed_id = vocab.special_pad_id;
} else { } else {
const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false); const std::vector<int> ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A
GGML_ASSERT(!ids.empty() && "model vocab missing newline token"); GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
vocab.linefeed_id = ids[0]; vocab.linefeed_id = ids[0];
} }