llama : fix command-r detokenization

2024-07-08 18:13:16 -04:00 · 2024-07-08 18:13:16 -04:00 · d6fe269ced
commit d6fe269ced
parent 31a1b0eeaa
1 changed files with 1 additions and 0 deletions
--- a/src/llama.cpp
+++ b/src/llama.cpp
@ -5407,6 +5407,7 @@ static void llm_load_vocab(
            } else if (
                tokenizer_pre == "command-r") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
+                vocab.tokenizer_clean_spaces = false;
            } else if (
                tokenizer_pre == "qwen2") {
                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2;