From 0c612e5e04a35d66b204f9a931f063e42b21af09 Mon Sep 17 00:00:00 2001 From: bobqianic <129547291+bobqianic@users.noreply.github.com> Date: Mon, 12 Feb 2024 11:05:47 +0000 Subject: [PATCH] Remove trailing whitespace in test-tokenizer-0-falcon.cpp --- tests/test-tokenizer-0-falcon.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-tokenizer-0-falcon.cpp b/tests/test-tokenizer-0-falcon.cpp index ed08c88af..065f36ba6 100644 --- a/tests/test-tokenizer-0-falcon.cpp +++ b/tests/test-tokenizer-0-falcon.cpp @@ -38,9 +38,9 @@ static const std::map> & k_tests() { { " Hello\n Hello" , { 466, 23090, 742, 23090, }, }, // { "\n =" , { 1212, 40, }, }, // This test has been disabled. Please re-enable it once the following issue has been resolved. - // Falcon requires extra steps during the pre-tokenization phase, as detailed in the pull request at https://github.com/ggerganov/llama.cpp/pull/5446. + // Falcon requires extra steps during the pre-tokenization phase, as detailed in the pull request at https://github.com/ggerganov/llama.cpp/pull/5446. // For more information on pre-tokenizers, you can refer to the Hugging Face documentation: - // https://huggingface.co/docs/tokenizers/en/api/pre-tokenizers#tokenizers.pre_tokenizers. + // https://huggingface.co/docs/tokenizers/en/api/pre-tokenizers#tokenizers.pre_tokenizers. // It's crucial that we address this issue ASAP to avoid any unforeseen outcomes. { "' era" , { 18, 4932, }, },