starcoder : fix pre-tokenizer

2024-05-02 11:00:15 +03:00 · 2024-05-02 11:00:15 +03:00 · cf00fe1ea3
commit cf00fe1ea3
parent 7053b261ab
22 changed files with 26 additions and 20 deletions
--- a/convert-hf-to-gguf-update.py
+++ b/convert-hf-to-gguf-update.py
@ -189,6 +189,7 @@ print("\n")
 # generate tests for each tokenizer model

 tests = [
+    "ied 4 ½ months"
    "",
    " ",
    "  ",
--- a/llama.cpp
+++ b/llama.cpp
@ -12235,6 +12235,11 @@ struct llm_tokenizer_bpe {
                        });
                        break;
                    case LLAMA_VOCAB_PRE_TYPE_STARCODER:
+                        word_collection = unicode_regex_split(text, {
+                            "\\p{N}",
+                            "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+                        });
+                        break;
                    case LLAMA_VOCAB_PRE_TYPE_GPT2:
                        word_collection = unicode_regex_split(text, {
                            "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
--- a/models/ggml-vocab-bert-bge.gguf.inp
+++ b/models/ggml-vocab-bert-bge.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-bert-bge.gguf.out
+++ b/models/ggml-vocab-bert-bge.gguf.out
@ -1,4 +1,4 @@
-
+ 29464 2094 1018 1092 2706



--- a/models/ggml-vocab-deepseek-coder.gguf.inp
+++ b/models/ggml-vocab-deepseek-coder.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-deepseek-coder.gguf.out
+++ b/models/ggml-vocab-deepseek-coder.gguf.out
@ -1,4 +1,4 @@
-
+ 1050 207 19 207 19192 4217
 207
 243
 315
--- a/models/ggml-vocab-deepseek-llm.gguf.inp
+++ b/models/ggml-vocab-deepseek-llm.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-deepseek-llm.gguf.out
+++ b/models/ggml-vocab-deepseek-llm.gguf.out
@ -1,4 +1,4 @@
-
+ 1052 207 19 207 19109 4223
 207
 243
 300
--- a/models/ggml-vocab-falcon.gguf.inp
+++ b/models/ggml-vocab-falcon.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-falcon.gguf.out
+++ b/models/ggml-vocab-falcon.gguf.out
@ -1,4 +1,4 @@
-
+ 878 204 31 3068 133 2137
 204
 258
 466
--- a/models/ggml-vocab-gpt-2.gguf.inp
+++ b/models/ggml-vocab-gpt-2.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-gpt-2.gguf.out
+++ b/models/ggml-vocab-gpt-2.gguf.out
@ -1,4 +1,4 @@
-
+ 798 604 25208 1933
 220
 220 220
 220 220 220
--- a/models/ggml-vocab-llama-bpe.gguf.inp
+++ b/models/ggml-vocab-llama-bpe.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-llama-bpe.gguf.out
+++ b/models/ggml-vocab-llama-bpe.gguf.out
@ -1,4 +1,4 @@
-
+ 1142 220 19 220 27154 4038
 220
 256
 262
--- a/models/ggml-vocab-llama-spm.gguf.inp
+++ b/models/ggml-vocab-llama-spm.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-llama-spm.gguf.out
+++ b/models/ggml-vocab-llama-spm.gguf.out
@ -1,4 +1,4 @@
-
+ 474 287 29871 29946 29871 30226 7378
 259
 1678
 268
--- a/models/ggml-vocab-mpt.gguf.inp
+++ b/models/ggml-vocab-mpt.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-mpt.gguf.out
+++ b/models/ggml-vocab-mpt.gguf.out
@ -1,4 +1,4 @@
-
+ 728 577 24142 2607
 209
 50276
 50275
--- a/models/ggml-vocab-phi-3.gguf.inp
+++ b/models/ggml-vocab-phi-3.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-phi-3.gguf.out
+++ b/models/ggml-vocab-phi-3.gguf.out
@ -1,4 +1,4 @@
-
+ 474 287 29871 29946 29871 30226 7378
 259
 1678
 268
--- a/models/ggml-vocab-starcoder.gguf.inp
+++ b/models/ggml-vocab-starcoder.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 
 __ggml_vocab_test__
--- a/models/ggml-vocab-starcoder.gguf.out
+++ b/models/ggml-vocab-starcoder.gguf.out
@ -1,4 +1,4 @@
-
+ 4850 244 57 244 162 159 17722
 244
 280
 283