starcoder : fix pre-tokenizer

2024-05-02 11:00:15 +03:00 · 2024-05-02 11:00:15 +03:00 · cf00fe1ea3
commit cf00fe1ea3
parent 7053b261ab
22 changed files with 26 additions and 20 deletions
--- a/convert-hf-to-gguf-update.py
+++ b/convert-hf-to-gguf-update.py
@ -189,6 +189,7 @@ print("\n")
 # generate tests for each tokenizer model
 tests = [
    "ied 4 ½ months"
    "",
    " ",
    "  ",
--- a/llama.cpp
+++ b/llama.cpp
@ -12235,6 +12235,11 @@ struct llm_tokenizer_bpe {
                        });
                        break;
                    case LLAMA_VOCAB_PRE_TYPE_STARCODER:
                        word_collection = unicode_regex_split(text, {
                            "\\p{N}",
                            "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
                        });
                        break;
                    case LLAMA_VOCAB_PRE_TYPE_GPT2:
                        word_collection = unicode_regex_split(text, {
                            "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
--- a/models/ggml-vocab-bert-bge.gguf.inp
+++ b/models/ggml-vocab-bert-bge.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-bert-bge.gguf.out
+++ b/models/ggml-vocab-bert-bge.gguf.out
@ -1,4 +1,4 @@
-
+ 29464 2094 1018 1092 2706
--- a/models/ggml-vocab-deepseek-coder.gguf.inp
+++ b/models/ggml-vocab-deepseek-coder.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-deepseek-coder.gguf.out
+++ b/models/ggml-vocab-deepseek-coder.gguf.out
@ -1,4 +1,4 @@
-
+ 1050 207 19 207 19192 4217
 207
 243
 315
--- a/models/ggml-vocab-deepseek-llm.gguf.inp
+++ b/models/ggml-vocab-deepseek-llm.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-deepseek-llm.gguf.out
+++ b/models/ggml-vocab-deepseek-llm.gguf.out
@ -1,4 +1,4 @@
-
+ 1052 207 19 207 19109 4223
 207
 243
 300
--- a/models/ggml-vocab-falcon.gguf.inp
+++ b/models/ggml-vocab-falcon.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-falcon.gguf.out
+++ b/models/ggml-vocab-falcon.gguf.out
@ -1,4 +1,4 @@
-
+ 878 204 31 3068 133 2137
 204
 258
 466
--- a/models/ggml-vocab-gpt-2.gguf.inp
+++ b/models/ggml-vocab-gpt-2.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-gpt-2.gguf.out
+++ b/models/ggml-vocab-gpt-2.gguf.out
@ -1,4 +1,4 @@
-
+ 798 604 25208 1933
 220
 220 220
 220 220 220
--- a/models/ggml-vocab-llama-bpe.gguf.inp
+++ b/models/ggml-vocab-llama-bpe.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-llama-bpe.gguf.out
+++ b/models/ggml-vocab-llama-bpe.gguf.out
@ -1,4 +1,4 @@
-
+ 1142 220 19 220 27154 4038
 220
 256
 262
--- a/models/ggml-vocab-llama-spm.gguf.inp
+++ b/models/ggml-vocab-llama-spm.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-llama-spm.gguf.out
+++ b/models/ggml-vocab-llama-spm.gguf.out
@ -1,4 +1,4 @@
-
+ 474 287 29871 29946 29871 30226 7378
 259
 1678
 268
--- a/models/ggml-vocab-mpt.gguf.inp
+++ b/models/ggml-vocab-mpt.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-mpt.gguf.out
+++ b/models/ggml-vocab-mpt.gguf.out
@ -1,4 +1,4 @@
-
+ 728 577 24142 2607
 209
 50276
 50275
--- a/models/ggml-vocab-phi-3.gguf.inp
+++ b/models/ggml-vocab-phi-3.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-phi-3.gguf.out
+++ b/models/ggml-vocab-phi-3.gguf.out
@ -1,4 +1,4 @@
-
+ 474 287 29871 29946 29871 30226 7378
 259
 1678
 268
--- a/models/ggml-vocab-starcoder.gguf.inp
+++ b/models/ggml-vocab-starcoder.gguf.inp
@ -1,4 +1,4 @@
-
+ied 4 ½ months
 __ggml_vocab_test__
 __ggml_vocab_test__
--- a/models/ggml-vocab-starcoder.gguf.out
+++ b/models/ggml-vocab-starcoder.gguf.out
@ -1,4 +1,4 @@
-
+ 4850 244 57 244 162 159 17722
 244
 280
 283
`@ -1,4 +1,4 @@`
		`1050 207 19 207 19192 4217`
	`207`	`207`
	`243`	`243`
	`315`	`315`
`@ -1,4 +1,4 @@`
		`1052 207 19 207 19109 4223`
	`207`	`207`
	`243`	`243`
	`300`	`300`
`@ -1,4 +1,4 @@`
		`878 204 31 3068 133 2137`
	`204`	`204`
	`258`	`258`
	`466`	`466`
`@ -1,4 +1,4 @@`
		`1142 220 19 220 27154 4038`
	`220`	`220`
	`256`	`256`
	`262`	`262`
`@ -1,4 +1,4 @@`
		`474 287 29871 29946 29871 30226 7378`
	`259`	`259`
	`1678`	`1678`
	`268`	`268`
`@ -1,4 +1,4 @@`
		`728 577 24142 2607`
	`209`	`209`
	`50276`	`50276`
	`50275`	`50275`
`@ -1,4 +1,4 @@`
		`4850 244 57 244 162 159 17722`
	`244`	`244`
	`280`	`280`
	`283`	`283`