From fc82541b1de0f625a0d3444563108c5b7e8bfd6b Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 17 Oct 2023 18:08:57 +0300 Subject: [PATCH] minor : comment + spacing --- llama.cpp | 2 +- llama.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index b571d4eb2..e894391f9 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6727,7 +6727,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & // by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer // and passing 'add space prefix' as bool argument // - auto raw_text = (special?"":" ") + fragment.raw_text.substr(fragment.offset, fragment.length); + auto raw_text = (special ? "" : " ") + fragment.raw_text.substr(fragment.offset, fragment.length); #ifdef PRETOKENIZERDEBUG fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); diff --git a/llama.h b/llama.h index b88ce9e42..b13f23123 100644 --- a/llama.h +++ b/llama.h @@ -516,6 +516,7 @@ extern "C" { /// @return Returns the number of tokens on success, no more than n_max_tokens /// @return Returns a negative number on failure - the number of tokens that would have been returned /// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. + /// Does not insert a leading space. LLAMA_API int llama_tokenize( const struct llama_model * model, const char * text,