diff --git a/src/llama.cpp b/src/llama.cpp index 7cdd27d86..3dc36d536 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -14888,6 +14888,9 @@ struct llm_tokenizer_ugm { std::string normalized; normalize(text, &normalized); size_t input_len = normalized.size(); + if (input_len == 0) { + return; + } // initialize score_sum to -FLT_MAX so it will be always lower than sums of token scores std::vector tokenization_results(input_len + 1, {0, 0, -FLT_MAX});