Remove invalid assert

This commit is contained in:
jaime-m-p 2024-08-13 16:30:30 +02:00
parent c240638374
commit 312c4322cc

View file

@ -710,7 +710,6 @@ struct llm_tokenizer_wpm {
continue; continue;
} }
assert (!categ.is_S());
if (cpt == 0 || cpt == 0xFFFD || categ.is_C()) { if (cpt == 0 || cpt == 0xFFFD || categ.is_C()) {
continue; continue;
} }