vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests : add test that fails with DeepSeek tokenizers

Browse source
This commit is contained in:
Georgi Gerganov 2024-05-02 11:46:20 +03:00
parent cf00fe1ea3
commit 3a461dbff3
No known key found for this signature in database
GPG key ID: BF970631944C16B7
22 changed files with 67 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

3
convert-hf-to-gguf-update.py
View file

@ -189,7 +189,8 @@ print("\n")
# generate tests for each tokenizer model # generate tests for each tokenizer model
tests = [ tests = [
"ied 4 ½ months" "ied 4 ½ months",
"Führer",
"", "",
" ", " ",
" ", " ",

9
llama.cpp
View file

@ -11952,7 +11952,7 @@ static bool llama_is_user_defined_token(const llama_vocab& vocab, llama_token id
static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) { static uint8_t llama_token_to_byte(const llama_vocab& vocab, llama_token id) {
GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE); GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE);
GGML_ASSERT(llama_is_byte_token(vocab, id)); GGML_ASSERT(llama_is_byte_token(vocab, id));
const auto& token_data = vocab.id_to_token.at(id); const auto & token_data = vocab.id_to_token.at(id);
switch (llama_vocab_get_type(vocab)) { switch (llama_vocab_get_type(vocab)) {
case LLAMA_VOCAB_TYPE_SPM: { case LLAMA_VOCAB_TYPE_SPM: {
auto buf = token_data.text.substr(3, 2); auto buf = token_data.text.substr(3, 2);
@ -17471,9 +17471,10 @@ int32_t llama_tokenize(
static std::string llama_decode_text(const std::string & text) { static std::string llama_decode_text(const std::string & text) {
std::string decoded_text; std::string decoded_text;
auto unicode_sequences = unicode_cpts_from_utf8(text);
for (auto & unicode_sequence : unicode_sequences) { const auto cpts = unicode_cpts_from_utf8(text);
decoded_text += unicode_utf8_to_byte(unicode_cpt_to_utf8(unicode_sequence)); for (const auto cpt : cpts) {
decoded_text += unicode_utf8_to_byte(unicode_cpt_to_utf8(cpt));
} }
return decoded_text; return decoded_text;

4
models/ggml-vocab-bert-bge.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-bert-bge.gguf.out
View file

@ -1,4 +1,6 @@
29464 2094 1018 1092 2706 29464 2094 1018 1092 2706
11865 17875

4
models/ggml-vocab-deepseek-coder.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-deepseek-coder.gguf.out
View file

@ -1,4 +1,6 @@
1050 207 19 207 19192 4217 1050 207 19 207 19192 4217
37 32009 71 6247
207 207
243 243
315 315

4
models/ggml-vocab-deepseek-llm.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-deepseek-llm.gguf.out
View file

@ -1,4 +1,6 @@
1052 207 19 207 19109 4223 1052 207 19 207 19109 4223
37 100014 71 6245
207 207
243 243
300 300

4
models/ggml-vocab-falcon.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-falcon.gguf.out
View file

@ -1,4 +1,6 @@
878 204 31 3068 133 2137 878 204 31 3068 133 2137
28611 132 30042
204 204
258 258
466 466

4
models/ggml-vocab-gpt-2.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-gpt-2.gguf.out
View file

@ -1,4 +1,6 @@
798 604 25208 1933 798 604 25208 1933
37 9116 71 11751
220 220
220 220 220 220
220 220 220 220 220 220

4
models/ggml-vocab-llama-bpe.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-llama-bpe.gguf.out
View file

@ -1,4 +1,6 @@
1142 220 19 220 27154 4038 1142 220 19 220 27154 4038
37 51853 261
220 220
256 256
262 262

4
models/ggml-vocab-llama-spm.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-llama-spm.gguf.out
View file

@ -1,4 +1,6 @@
474 287 29871 29946 29871 30226 7378 474 287 29871 29946 29871 30226 7378
383 4000 261
259 259
1678 1678
268 268

4
models/ggml-vocab-mpt.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-mpt.gguf.out
View file

@ -1,4 +1,6 @@
728 577 24142 2607 728 577 24142 2607
39 26288 6554
209 209
50276 50276
50275 50275

4
models/ggml-vocab-phi-3.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-phi-3.gguf.out
View file

@ -1,4 +1,6 @@
474 287 29871 29946 29871 30226 7378 474 287 29871 29946 29871 30226 7378
383 4000 261
259 259
1678 1678
268 268

4
models/ggml-vocab-starcoder.gguf.inp
View file

@ -1,4 +1,8 @@
ied 4 ½ months ied 4 ½ months
__ggml_vocab_test__
Führer
__ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__
__ggml_vocab_test__ __ggml_vocab_test__

2
models/ggml-vocab-starcoder.gguf.out
View file

@ -1,4 +1,6 @@
4850 244 57 244 162 159 17722 4850 244 57 244 162 159 17722
75 2022 3943 284
244 244
280 280
283 283