'viking' detokenizer clean spaces

2024-07-04 22:30:48 +02:00 · 2024-07-04 22:30:48 +02:00 · 8f5e1e0c76
commit 8f5e1e0c76
parent 8072089c4e
1 changed files with 1 additions and 0 deletions
--- a/src/llama.cpp
+++ b/src/llama.cpp
@ -5158,6 +5158,7 @@ static void llm_load_vocab(
            } else if (
                tokenizer_pre == "viking") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_VIKING;
+                vocab.tokenizer_clean_spaces = false;
            } else if (
                tokenizer_pre == "jais") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS;