From 735ffe3d2f8f57fd0b1066e48ff690dd31b67854 Mon Sep 17 00:00:00 2001
From: Jared Van Bortel <jared@nomic.ai>
Date: Tue, 14 Nov 2023 16:48:33 -0500
Subject: [PATCH] Revert "dont add space when using special tokens"

This reverts commit 1c28116de410ad689fdf26d637be6f0530107cb0.
---
 llama.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 01522fdb4..fe4ff7369 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -6283,7 +6283,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
                         //  by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer
                         //  and passing 'add space prefix' as bool argument
                         //
-                        auto raw_text = (special ? "" : " ") + fragment.raw_text.substr(fragment.offset, fragment.length);
+                        auto raw_text = " " + fragment.raw_text.substr(fragment.offset, fragment.length);
 
 #ifdef PRETOKENIZERDEBUG
                         fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str());