falcon : fix regex

2024-05-02 11:52:50 +03:00 · 2024-05-02 11:52:50 +03:00 · 3275e60f57
commit 3275e60f57
parent 3a461dbff3
1 changed files with 1 additions and 2 deletions
--- a/llama.cpp
+++ b/llama.cpp
@ -12212,14 +12212,13 @@ struct llm_tokenizer_bpe {
                            "\\s?\\p{L}+",
                            "\\s?\\p{P}+",
                            "[一-龥ࠀ-一가-퟿]+",
-                            "\\p{N}+",
+                            "\\p{N}",
                        });
                        break;
                    case LLAMA_VOCAB_PRE_TYPE_FALCON:
                        word_collection = unicode_regex_split(text, {
                            "[\\p{P}\\$\\+<=>\\^~\\|]+",
                            "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
-                            "\\p{N}+",
                            "[0-9][0-9][0-9]",
                        });
                        break;