vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: keep refactoring non-breaking

Browse source
This commit is contained in:
Joan Martinez 2024-05-13 10:28:26 +02:00
parent 8957cacd98
commit fb83012096
1 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama.cpp
View file

@ -4423,6 +4423,8 @@ static void llm_load_vocab(
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER;
} else if (
tokenizer_pre == "gpt-2" ||
tokenizer_pre == "jina-es" ||
tokenizer_pre == "jina-de" ||
tokenizer_pre == "jina-v2-es" ||
tokenizer_pre == "jina-v2-de") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;