From 974d40b513b8c86491db7db06758e31caa666dbd Mon Sep 17 00:00:00 2001
From: jaime-m-p <>
Date: Thu, 13 Jun 2024 20:40:56 +0200
Subject: [PATCH] Fix 'jina-v2' per token attributes

---
 llama.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 50ea36ea7..6422587c9 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -4927,7 +4927,7 @@ static void llm_load_vocab(
         );
 
         // set attributes by model/tokenizer name
-        if (_contains_any(tokenizer_pre, {"jina-v2-"})) {
+        if (_contains_any(tokenizer_pre, {"jina-v2-de", "jina-v2-es", "jina-v2-code"})) {
             _set_token_attr("<mask>", LLAMA_TOKEN_ATTR_LSTRIP, true);
         } else if (_contains_any(model_name, {"phi-3", "phi3"})) {
             for (auto id : vocab.cache_special_tokens) {