Remove invalid assert

2024-08-13 16:30:30 +02:00 · 2024-08-13 16:30:30 +02:00 · 312c4322cc
commit 312c4322cc
parent c240638374
1 changed files with 0 additions and 1 deletions
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@ -710,7 +710,6 @@ struct llm_tokenizer_wpm {
                continue;
            }

-            assert (!categ.is_S());
            if (cpt == 0 || cpt == 0xFFFD || categ.is_C()) {
                continue;
            }