From 51e933a962415196f015380eb5eb58764d006a5d Mon Sep 17 00:00:00 2001
From: jaime-m-p <>
Date: Sat, 25 May 2024 04:32:45 +0200
Subject: [PATCH] Fix falcon punctuation regex

---
 llama.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llama.cpp b/llama.cpp
index 046449130..bb5cd9fe2 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -12315,7 +12315,7 @@ struct llm_tokenizer_bpe {
                 break;
             case LLAMA_VOCAB_PRE_TYPE_FALCON:
                 regex_exprs = {
-                    "[\\p{P}\\$\\+<=>\\^~\\|]+",
+                    "[\\p{P}\\$\\+<=>\\^~\\|`]+",
                     "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
                     "[0-9][0-9][0-9]",
                 };