llama : use new pre-tokenizer type

2024-04-26 20:08:28 +03:00 · 2024-04-26 20:08:28 +03:00 · 43e12ce8e5
commit 43e12ce8e5
parent 9b4d63ae53
12 changed files with 87 additions and 44 deletions
--- a/tests/test-tokenizer-0-bpe.py
+++ b/tests/test-tokenizer-0-bpe.py
@ -27,6 +27,8 @@ tests = [
    "   ",
    "\t",
    "\n",
+    "\n\n",
+    "\n\n\n",
    "\t\n",
    "Hello world",
    " Hello world",