vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: llama-3 ignore_merges

Browse source
This commit is contained in:
Haoxiang Fei 2024-05-10 15:24:35 +08:00
parent e849648888
commit c21d5e13fe
1 changed files with 14 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama.cpp
View file

@ -12292,6 +12292,20 @@ struct llm_tokenizer_bpe {
symbols_final.clear();
for (auto & word : word_collection) {
if (vocab.token_to_id.find(word) != vocab.token_to_id.end()) {
llm_symbol sym;
sym.text = word.c_str();
sym.n = word.size();
sym.prev = final_prev_index;
sym.next = -1;
if (final_prev_index != -1) {
symbols_final[final_prev_index].next = symbols_final.size();
}
symbols_final.emplace_back(sym);
final_prev_index = symbols_final.size() - 1;
continue;
}
work_queue = llm_bigram_bpe::queue();
symbols.clear();