Fix falcon punctuation regex

2024-05-25 04:32:45 +02:00 · 2024-05-25 04:32:45 +02:00 · 51e933a962
commit 51e933a962
parent 0794b77714
1 changed files with 1 additions and 1 deletions
--- a/llama.cpp
+++ b/llama.cpp
@ -12315,7 +12315,7 @@ struct llm_tokenizer_bpe {
                break;
            case LLAMA_VOCAB_PRE_TYPE_FALCON:
                regex_exprs = {
-                    "[\\p{P}\\$\\+<=>\\^~\\|]+",
+                    "[\\p{P}\\$\\+<=>\\^~\\|`]+",
                    "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
                    "[0-9][0-9][0-9]",
                };