From e8185370276ef43d0a98265c97704fb86c02fa40 Mon Sep 17 00:00:00 2001 From: Howard Su Date: Sun, 2 Jul 2023 22:06:03 +0800 Subject: [PATCH 1/5] [llama] Add resegment post processing of tokenizer --- llama.cpp | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/llama.cpp b/llama.cpp index 7419b03b6..c9488dc6f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1771,21 +1771,35 @@ struct llama_tokenizer { for (int i = 0; i != -1; i = symbols_[i].next) { auto & symbol = symbols_[i]; - auto token = vocab_.token_to_id.find(std::string(symbol.text, symbol.n)); - - if (token == vocab_.token_to_id.end()) { - // output any symbols that did not form tokens as bytes. - for (int j = 0; j < (int) symbol.n; ++j) { - llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; - output.push_back(token_id); - } - } else { - output.push_back((*token).second); - } + resegment(symbol, output); } } private: + void resegment(llama_sp_symbol &symbol, std::vector &output) { + auto text = std::string(symbol.text, symbol.n); + auto token = vocab_.token_to_id.find(text); + + if (token != vocab_.token_to_id.end()) { + output.push_back((*token).second); + return; + } + + const auto p = rev_merge.find(text); + + if (p == rev_merge.end()) { + // output any symbols that did not form tokens as bytes. + for (int j = 0; j < (int) symbol.n; ++j) { + llama_vocab::id token_id = static_cast(symbol.text[j]) + 3; + output.push_back(token_id); + } + return; + } + + resegment(symbols_[p->second.first], output); + resegment(symbols_[p->second.second], output); + } + void try_add_bigram(int left, int right) { if (left == -1 || right == -1) { return; @@ -1810,11 +1824,14 @@ private: bigram.score = tok_score.score; bigram.size = text.size(); work_queue_.push(bigram); + + rev_merge[text] = std::make_pair(left, right); } const llama_vocab & vocab_; std::vector symbols_; llama_sp_bigram::queue work_queue_; + std::map > rev_merge; }; static std::vector llama_tokenize(const llama_vocab & vocab, const std::string & text, bool bos) { From 6caa06638f7c08266351673b7e2e8beae528aa55 Mon Sep 17 00:00:00 2001 From: Howard Su Date: Sun, 2 Jul 2023 22:06:10 +0800 Subject: [PATCH 2/5] Add tests --- tests/test-tokenizer-0.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 20abe7100..2548416d8 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -14,6 +14,7 @@ static const std::map> & k_tests() { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, + { ">>>>ANSWER<<", { 1, 6778, 6778, 2190, 23066, 1001, 9314,}, }, }; return _k_tests; }; @@ -94,6 +95,38 @@ int main(int argc, char **argv) { } } +#if 0 + // how many tokens would not tokenize to themselves + for (llama_token i = 1; i < llama_n_vocab(ctx); i++) + { + const char* str = llama_token_to_str(ctx, i); + std::vector res(100); + + const int n = llama_tokenize(ctx, str, res.data(), int(res.size()), false); + res.resize(n); + + for (const auto & t : res) + { + //if (t == 1) continue; + + if (t != i) { + fprintf(stderr, "%s : failed test: '%s'\n", __func__, str); + fprintf(stderr, "%s : expected tokens: %d\n", __func__, i); + fprintf(stderr, "%s : got tokens: ", __func__); + for (const auto & t : res) { + fprintf(stderr, "%6d, ", t); + } + for (const auto & t : res) { + fprintf(stderr, "%s|", llama_token_to_str(ctx, t)); + } + + fprintf(stderr, "\n"); + } + } + + } +#endif + llama_free_model(model); llama_free(ctx); From 751e51cedad4e705ae93a4655289c76e60b1bd3a Mon Sep 17 00:00:00 2001 From: Howard Su Date: Mon, 3 Jul 2023 09:51:04 +0800 Subject: [PATCH 3/5] special test --- tests/test-tokenizer-0.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 2548416d8..5e4bc4a78 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -8,13 +8,14 @@ static const std::map> & k_tests() { static std::map> _k_tests = { - { "Hello World", { 1, 10994, 2787, }, }, - { " Hello World", { 1, 15043, 2787, }, }, - { " Hello World!", { 1, 15043, 2787, 29991, }, }, - { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, - { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, - { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, - { ">>>>ANSWER<<", { 1, 6778, 6778, 2190, 23066, 1001, 9314,}, }, + // { "Hello World", { 1, 10994, 2787, }, }, + // { " Hello World", { 1, 15043, 2787, }, }, + // { " Hello World!", { 1, 15043, 2787, 29991, }, }, + // { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, + // { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, + // { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, + // { ">>>>ANSWER<<", { 1, 6778, 6778, 2190, 23066, 1001, 9314,}, }, + { "\xe2\x96\x81\xe2\x80\x93", { 1, 100, }, }, }; return _k_tests; }; From ca150b7725af5f68889e38d0c0bce55d493e828c Mon Sep 17 00:00:00 2001 From: Howard Su Date: Tue, 4 Jul 2023 18:15:25 +0800 Subject: [PATCH 4/5] More tests --- tests/test-tokenizer-0.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 5e4bc4a78..5575e26ec 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -8,14 +8,14 @@ static const std::map> & k_tests() { static std::map> _k_tests = { - // { "Hello World", { 1, 10994, 2787, }, }, - // { " Hello World", { 1, 15043, 2787, }, }, - // { " Hello World!", { 1, 15043, 2787, 29991, }, }, - // { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, - // { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, - // { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, - // { ">>>>ANSWER<<", { 1, 6778, 6778, 2190, 23066, 1001, 9314,}, }, - { "\xe2\x96\x81\xe2\x80\x93", { 1, 100, }, }, + { "Hello World", { 1, 10994, 2787, }, }, + { " Hello World", { 1, 15043, 2787, }, }, + { " Hello World!", { 1, 15043, 2787, 29991, }, }, + { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, + { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, + { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, + { "\xe6\x88\x91\xe4\xbb\xac\xe5\xa4\xa7\xe5\xae\xb6\xe4\xb8\x80\xe8\xb5\xb7", { 1, 30672, 31381, 30257, 30613, 30287, 31558, }, }, + { ">>>>ANSWER<<", {1, 5099, 6778, 2190, 23066, 1001, 9314}, }, }; return _k_tests; }; @@ -84,11 +84,19 @@ int main(int argc, char **argv) { fprintf(stderr, "%6d, ", t); } fprintf(stderr, "\n"); + for (const auto & t : test_kv.second) { + fprintf(stderr, "%7s ", llama_token_to_str(ctx, t)); + } + fprintf(stderr, "\n"); fprintf(stderr, "%s : got tokens: ", __func__); for (const auto & t : res) { fprintf(stderr, "%6d, ", t); } fprintf(stderr, "\n"); + for (const auto & t : res) { + fprintf(stderr, "%7s ", llama_token_to_str(ctx, t)); + } + fprintf(stderr, "\n"); llama_free_model(model); llama_free(ctx); From 5f04a5d877e909539fdc1fd7a8e06836fa1c8c36 Mon Sep 17 00:00:00 2001 From: Howard Su Date: Tue, 4 Jul 2023 21:08:48 +0800 Subject: [PATCH 5/5] Add test --- tests/test-tokenizer-0.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 5575e26ec..a5f0770bf 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -15,7 +15,7 @@ static const std::map> & k_tests() { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, { "\xe6\x88\x91\xe4\xbb\xac\xe5\xa4\xa7\xe5\xae\xb6\xe4\xb8\x80\xe8\xb5\xb7", { 1, 30672, 31381, 30257, 30613, 30287, 31558, }, }, - { ">>>>ANSWER<<", {1, 5099, 6778, 2190, 23066, 1001, 9314}, }, + { " >>>>ANSWER<<", {1, 5099, 6778, 2190, 23066, 1001, 9314}, }, }; return _k_tests; };