vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'refactor-jina-rename' of https://github.com/JoanFM/llama.cpp into feat-jina-embeddings-v2-zh

Browse source
This commit is contained in:
Joan Martinez 2024-05-13 10:29:55 +02:00
commit ea0f7df2fb
3 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama.cpp
View file

@ -4423,6 +4423,8 @@ static void llm_load_vocab(
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STARCODER;
} else if (
tokenizer_pre == "gpt-2" ||
tokenizer_pre == "jina-es" ||
tokenizer_pre == "jina-de" ||
tokenizer_pre == "jina-v2-es" ||
tokenizer_pre == "jina-v2-de") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2;