From fb1fef99622c6c8004e974b6d2a765a758e96400 Mon Sep 17 00:00:00 2001
From: Gilad S <giladgd@users.noreply.github.com>
Date: Sun, 2 Jun 2024 19:58:43 +0300
Subject: [PATCH] fix: don't add space after special tokens when using SPM

---
 llama.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 841be1de7..a38a34a6f 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -13393,7 +13393,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                         }
 
                         if (vocab.add_space_prefix) {
-                            if (!output.size() || is_prev_special) {  // prefix with space if first token
+                            if (!output.size() && !is_prev_special) {  // prefix with space if first token
                                 raw_text = " " + raw_text;
                             }
                         }