From e41209a95f800b169a8439e4122062d53df80ce5 Mon Sep 17 00:00:00 2001
From: goerch <jhr.walter@t-online.de>
Date: Thu, 14 Sep 2023 21:26:08 +0200
Subject: [PATCH] llama_tokenize should accept strings containing NUL now

---
 common/common.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/common.cpp b/common/common.cpp
index 3898fc3ce..fd47dbc2d 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -804,10 +804,10 @@ std::vector<llama_token> llama_tokenize(
     // upper limit for the number of tokens
     int n_tokens = text.length() + add_bos;
     std::vector<llama_token> result(n_tokens);
-    n_tokens = llama_tokenize(ctx, text.c_str(), text.length(), result.data(), result.size(), add_bos);
+    n_tokens = llama_tokenize(ctx, text.data(), text.length(), result.data(), result.size(), add_bos);
     if (n_tokens < 0) {
         result.resize(-n_tokens);
-        int check = llama_tokenize(ctx, text.c_str(), text.length(), result.data(), result.size(), add_bos);
+        int check = llama_tokenize(ctx, text.data(), text.length(), result.data(), result.size(), add_bos);
         GGML_ASSERT(check == -n_tokens);
     } else {
         result.resize(n_tokens);