From bb6a58d0c3d9aced445ac352d511f892710a8d33 Mon Sep 17 00:00:00 2001
From: goerch <jhr.walter@t-online.de>
Date: Sun, 6 Aug 2023 13:35:27 +0200
Subject: [PATCH] Simplifying an expression.

---
 llama.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 797bbd760..4e3e2a922 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -4096,7 +4096,8 @@ int llama_tokenize_with_model(
                  llama_token * tokens,
                          int   n_max_tokens,
                         bool   add_bos) {
-    auto res = llama_tokenize(model->vocab, text, add_bos, llama_vocab_type(model->vocab) == "spm");
+    auto escape = llama_vocab_type(model->vocab) == "spm";
+    auto res = llama_tokenize(model->vocab, text, add_bos, escape);
 
     if (n_max_tokens < (int) res.size()) {
         return -((int) res.size());