From 0ae2860faa8dd754412066d4dc32102c06479adc Mon Sep 17 00:00:00 2001 From: jaime-m-p <167997752+jaime-m-p@users.noreply.github.com> Date: Sun, 19 May 2024 21:13:20 +0200 Subject: [PATCH] Apply suggestions from code review Co-authored-by: Georgi Gerganov --- llama.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 0e8a850ed..f3fd68f0b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12820,13 +12820,13 @@ static std::vector llama_tokenize_internal(const llama_vocab & while (isspace(raw_text[num_whitespaces])) { num_whitespaces++; } - if(num_whitespaces == raw_text.size()) { + if (num_whitespaces == raw_text.size()) { continue; // skip if all whitespaces } raw_text = raw_text.substr(num_whitespaces); } - if(vocab.add_space_prefix) { + if (vocab.add_space_prefix) { if (!output.size() || is_prev_special) { // prefix with space if first token raw_text = " " + raw_text; }