diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 5eeae0585..6192fd195 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -710,7 +710,6 @@ struct llm_tokenizer_wpm { continue; } - assert (!categ.is_S()); if (cpt == 0 || cpt == 0xFFFD || categ.is_C()) { continue; }