From d0c8d14eaec949382e98ad2153d46be377f3d300 Mon Sep 17 00:00:00 2001 From: goerch Date: Tue, 10 Oct 2023 07:41:49 +0200 Subject: [PATCH] Don't add bos token in test --- tests/test-tokenizer-0-falcon.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-tokenizer-0-falcon.cpp b/tests/test-tokenizer-0-falcon.cpp index 25038401c..a4e9d2b91 100644 --- a/tests/test-tokenizer-0-falcon.cpp +++ b/tests/test-tokenizer-0-falcon.cpp @@ -157,7 +157,7 @@ int main(int argc, char **argv) { fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); - const std::vector res = llama_tokenize(ctx, text, true); + const std::vector res = llama_tokenize(ctx, text, false); fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());