diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index b3a344ef3..92bbfa199 100644 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -46,8 +46,8 @@ else: # TODO: add models here, base models preferred models = [ - { "name": "llama-v2", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", }, - { "name": "llama-v3", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", }, + { "name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", }, + { "name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", }, { "name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", }, { "name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", }, { "name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", }, @@ -64,7 +64,7 @@ def download_file_with_auth(url, token, save_path): if response.status_code == 200: with open(save_path, 'wb') as f: f.write(response.content) - print("File downloaded successfully.") + print(f"File {save_path} downloaded successfully") else: print(f"Failed to download file. Status code: {response.status_code}") @@ -82,6 +82,10 @@ for model in models: print(f"Downloading {name} to models/tokenizers/{name}") + url = f"{repo}/raw/main/config.json" + save_path = f"models/tokenizers/{name}/config.json" + download_file_with_auth(url, token, save_path) + url = f"{repo}/raw/main/tokenizer.json" save_path = f"models/tokenizers/{name}/tokenizer.json" download_file_with_auth(url, token, save_path) @@ -219,7 +223,7 @@ tests = [ "333333333", ] -# write the tests in ./models/test-vocab-inp.txt +# write the tests to ./models/ggml-vocab-{name}.gguf.inp # the format is: # # test0 @@ -229,14 +233,7 @@ tests = [ # ... # -with open(f"models/test-vocab-inp.txt", "w") as f: - for text in tests: - f.write(f"{text}") - f.write("\n__ggml_vocab_test__\n") - -print("Tests written in ./models/test-vocab-inp.txt") - -# with each model, encode all tests and write the results in ./models/test-vocab-out-{name}.txt +# with each model, encode all tests and write the results in ./models/ggml-vocab-{name}.gguf.out # for each test, write the resulting tokens on a separate line for model in models: @@ -247,11 +244,27 @@ for model in models: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}") - with open(f"models/test-vocab-out-{name}.txt", "w") as f: + with open(f"models/ggml-vocab-{name}.gguf.inp", "w") as f: for text in tests: - res = tokenizer.encode(text) + f.write(f"{text}") + f.write("\n__ggml_vocab_test__\n") + + with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f: + for text in tests: + res = tokenizer.encode(text, add_special_tokens=False) for r in res: f.write(f" {r}") f.write("\n") - print(f"Test results for {name} written in ./models/test-vocab-out-{name}.txt") + print(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*") + +# generate commands for creating vocab files + +print("\nRun the following commands to generate the vocab files for testing:\n") + +for model in models: + name = model["name"] + + print(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only") + +print("\n") diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 5370f3dfd..cc1b39339 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -283,7 +283,7 @@ class Model(ABC): # don't do this manually - use the convert-hf-to-gguf-update.py script! if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5": # ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B - res = "llama-v3" + res = "llama-bpe" if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754": # ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base res = "deepseek-llm" diff --git a/llama.cpp b/llama.cpp index 98316161c..7358b8d46 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4339,8 +4339,9 @@ static void llm_load_vocab( tokenizer_pre == "default") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT; } else if ( - tokenizer_pre == "llama3" || - tokenizer_pre == "llama-v3") { + tokenizer_pre == "llama3" || + tokenizer_pre == "llama-v3" || + tokenizer_pre == "llama-bpe") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3; } else if ( tokenizer_pre == "deepseek-llm") { @@ -12583,7 +12584,7 @@ static std::vector llama_tokenize_internal(const llama_vocab & } break; case LLAMA_VOCAB_TYPE_BPE: { - if (add_special && vocab.special_add_bos == 1) { + if (add_special && vocab.special_add_bos != 0) { GGML_ASSERT(vocab.special_bos_id != -1); output.push_back(vocab.special_bos_id); } diff --git a/models/ggml-vocab-deepseek-coder.gguf b/models/ggml-vocab-deepseek-coder.gguf index 60de2c4b5..6728cd747 100644 Binary files a/models/ggml-vocab-deepseek-coder.gguf and b/models/ggml-vocab-deepseek-coder.gguf differ diff --git a/models/ggml-vocab-deepseek-llm.gguf b/models/ggml-vocab-deepseek-llm.gguf index be0931fdf..5d66091c4 100644 Binary files a/models/ggml-vocab-deepseek-llm.gguf and b/models/ggml-vocab-deepseek-llm.gguf differ diff --git a/models/ggml-vocab-llama-v3.gguf b/models/ggml-vocab-llama-v3.gguf deleted file mode 100644 index fedd738a0..000000000 Binary files a/models/ggml-vocab-llama-v3.gguf and /dev/null differ diff --git a/models/ggml-vocab-llama.gguf b/models/ggml-vocab-llama.gguf deleted file mode 100644 index 568ffdc16..000000000 Binary files a/models/ggml-vocab-llama.gguf and /dev/null differ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ca7bbb6ec..a9f5bd34b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -65,21 +65,16 @@ function(llama_target_and_test source) set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL}) endfunction() -# llama_target_and_test(test-double-float.cpp) # SLOW -llama_target_and_test(test-quantize-fns.cpp) -llama_target_and_test(test-quantize-perf.cpp) -llama_target_and_test(test-sampling.cpp) -llama_target_and_test(test-chat-template.cpp) +# build test-tokenizer-0 target once and add many tests +add_executable(test-tokenizer-0 test-tokenizer-0.cpp get-model.cpp) +target_link_libraries(test-tokenizer-0 PRIVATE common) +install(TARGETS test-tokenizer-0 RUNTIME) -llama_target_and_test(test-tokenizer-0-llama.cpp NAME test-tokenizer-0-llama ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) -llama_target_and_test(test-tokenizer-0-llama-v3.cpp NAME test-tokenizer-0-llama-v3 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-v3.gguf) -llama_target_and_test(test-tokenizer-0-falcon.cpp NAME test-tokenizer-0-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) - -llama_target_and_test(test-tokenizer-0-deepseek-coder.cpp NAME test-tokenizer-0-deepseek-coder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-coder.gguf) -llama_target_and_test(test-tokenizer-0-deepseek-llm.cpp NAME test-tokenizer-0-deepseek-llm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-llm.gguf) - -llama_target_and_test(test-tokenizer-1-llama.cpp NAME test-tokenizer-1-llama ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf) -llama_target_and_test(test-tokenizer-1-llama.cpp NAME test-tokenizer-1-baichuan ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-baichuan.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-spm.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-bpe.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-llm.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-coder.gguf) # build test-tokenizer-1-bpe target once and add many tests add_executable(test-tokenizer-1-bpe test-tokenizer-1-bpe.cpp get-model.cpp) @@ -96,9 +91,19 @@ llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-starcoder ARGS ${CMAKE_CUR llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt2 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt2.gguf) #llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-bloom ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-bloom.gguf) # BIG +# build test-tokenizer-1-spm target once and add many tests +add_executable(test-tokenizer-1-spm test-tokenizer-1-spm.cpp get-model.cpp) +target_link_libraries(test-tokenizer-1-spm PRIVATE common) +install(TARGETS test-tokenizer-1-spm RUNTIME) +llama_target_and_test(test-tokenizer-1-spm NAME test-tokenizer-1-llama-spm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-spm.gguf) +llama_target_and_test(test-tokenizer-1-spm NAME test-tokenizer-1-baichuan ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-baichuan.gguf) - +# llama_target_and_test(test-double-float.cpp) # SLOW +llama_target_and_test(test-quantize-fns.cpp) +llama_target_and_test(test-quantize-perf.cpp) +llama_target_and_test(test-sampling.cpp) +llama_target_and_test(test-chat-template.cpp) llama_target_and_test(test-grammar-parser.cpp) llama_target_and_test(test-llama-grammar.cpp) diff --git a/tests/test-tokenizer-0-deepseek-coder.cpp b/tests/test-tokenizer-0-deepseek-coder.cpp deleted file mode 100644 index a3dc0047d..000000000 --- a/tests/test-tokenizer-0-deepseek-coder.cpp +++ /dev/null @@ -1,188 +0,0 @@ -#include "llama.h" -#include "common.h" -#include "console.h" - -#include -#include -#include -#include -#include - -// generate using test-tokenizer-0-falcon.py -static const std::map> & k_tests() { - static std::map> _k_tests = { - { "" , { }, }, - { " " , { 207, }, }, - { " " , { 243, }, }, - { " " , { 315, }, }, - { "\t" , { 184, }, }, - { "\n" , { 185, }, }, - { "\t\n" , { 184, 185, }, }, - { "Hello world" , { 17535, 1835, }, }, - { " Hello world" , { 414, 9489, 1835, }, }, - { "Hello World" , { 17535, 5414, }, }, - { " Hello World" , { 414, 9489, 5414, }, }, - { " Hello World!" , { 414, 9489, 5414, 0, }, }, - { "Hello, world!" , { 17535, 11, 1835, 0, }, }, - { " Hello, world!" , { 414, 9489, 11, 1835, 0, }, }, - { " this is πŸ¦™.cpp" , { 437, 317, 12394, 99, 234, 13, 14789, }, }, - { "w048 7tuijk dsdfhu" , { 86, 15, 19, 23, 207, 22, 83, 3963, 27659, 26078, 3934, 14072, }, }, - { "Π½Π΅Ρ‰ΠΎ Π½Π° Π‘ΡŠΠ»Π³Π°Ρ€ΡΠΊΠΈ" , { 1593, 6478, 616, 2251, 14994, }, }, - { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰" , { 155, 239, 209, 155, 239, 114, 155, 239, 228, 155, 240, 220, 155, 239, 224, 155, 240, 211, 155, 239, 231, 155, 239, 115, 155, 239, 240, 155, 240, 210, 155, 239, 240, 155, 239, 95, 155, 239, 114, 155, 239, 214, 155, 239, 210, 155, 239, 236, 155, 239, 214, 155, 240, 210, 155, 239, 218, }, }, - { "πŸš€ (normal) πŸ˜Άβ€πŸŒ«οΈ (multiple emojis concatenated) βœ… (only emoji that has its own token)", { 10047, 235, 209, 334, 8760, 8, 12394, 233, 114, 350, 222, 10047, 221, 104, 169, 116, 224, 334, 4684, 3909, 992, 24330, 262, 29651, 612, 8, 207, 156, 237, 214, 334, 5950, 992, 78, 12896, 344, 638, 891, 1372, 10736, 8, }, }, - { "Hello" , { 17535, }, }, - { " Hello" , { 414, 9489, }, }, - { " Hello" , { 207, 414, 9489, }, }, - { " Hello" , { 243, 414, 9489, }, }, - { " Hello" , { 315, 414, 9489, }, }, - { " Hello\n Hello" , { 315, 414, 9489, 185, 315, 414, 9489, }, }, - { "\n =" , { 185, 405, }, }, - { "' era" , { 6, 2895, }, }, - { "Hello, y'all! How are you 😁 ?ζˆ‘ζƒ³εœ¨appleε·₯作1314151倩～", { 17535, 11, 320, 6, 435, 0, 1717, 417, 340, 12394, 233, 210, 3015, 19100, 608, 9413, 2668, 16, 18, 16, 19, 16, 20, 16, 1393, 169, 121, 239, }, }, - - }; - - return _k_tests; -} - -int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); - return 1; - } - - const std::string fname = argv[1]; - - std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; - } - - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); - - llama_model * model; - llama_context * ctx; - - llama_backend_init(); - - // load the vocab - { - auto mparams = llama_model_default_params(); - - mparams.vocab_only = true; - - model = llama_load_model_from_file(fname.c_str(), mparams); - - if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - return 1; - } - - auto cparams = llama_context_default_params(); - - ctx = llama_new_context_with_model(model, cparams); - - if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - llama_free_model(model); - return 1; - } - } - - if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) { - fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__); - llama_free_model(model); - llama_free(ctx); - return 2; - } - -#ifdef _WIN32 - // We need this for unicode console support - console::init(false, false); - atexit([]() { console::cleanup(); }); -#endif - - bool success = true; - - for (const auto & test_kv : k_tests()) { - const std::vector res = llama_tokenize(ctx, test_kv.first, false); - - printf("\n"); - printf("src: '%s'\n", test_kv.first.c_str()); - printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str()); - printf("tok: "); - for (const auto & tok : res) { - printf("%d ", tok); - } - printf("\n"); - - bool correct = res.size() == test_kv.second.size(); - for (int i = 0; i < (int) res.size() && correct; ++i) { - if (test_kv.second[i] != res[i]) { - correct = false; - } - } - - if (!correct) { - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); - fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, - llama_detokenize_bpe(ctx, res).c_str(), - llama_detokenize_bpe(ctx, test_kv.second).c_str()); - fprintf(stderr, "%s : expected tokens: ", __func__); - for (const auto & t : test_kv.second) { - fprintf(stderr, "%6d, ", t); - } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : got tokens: ", __func__); - for (const auto & t : res) { - fprintf(stderr, "%6d, ", t); - } - fprintf(stderr, "\n"); - - success = false; - } - } - - if (!fname_text.empty()) { - fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); - - std::string text; - { - std::ifstream ifs(fname_text); - if (!ifs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); - return 1; - } - text = std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator()); - } - - fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); - - const std::vector res = llama_tokenize(ctx, text, false); - - fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); - - { - const std::string fname_out = fname_text + ".tokcpp"; - - std::ofstream ofs(fname_out); - if (!ofs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); - return 1; - } - - for (const auto & tok : res) { - ofs << tok << " '" << llama_detokenize_bpe(ctx, std::vector{tok}) << "'" << std::endl; - } - } - - fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); - } - - llama_free_model(model); - llama_free(ctx); - - llama_backend_free(); - - return success ? 0 : 3; -} diff --git a/tests/test-tokenizer-0-deepseek-llm.cpp b/tests/test-tokenizer-0-deepseek-llm.cpp deleted file mode 100644 index e21d16c88..000000000 --- a/tests/test-tokenizer-0-deepseek-llm.cpp +++ /dev/null @@ -1,186 +0,0 @@ -#include "llama.h" -#include "common.h" -#include "console.h" - -#include -#include -#include -#include -#include - -// generate using test-tokenizer-0-falcon.py -static const std::map> & k_tests() { - static std::map> _k_tests = { - { "" , { }, }, - { " " , { 207, }, }, - { " " , { 243, }, }, - { " " , { 300, }, }, - { "\t" , { 184, }, }, - { "\n" , { 185, }, }, - { "\t\n" , { 184, 185, }, }, - { "Hello world" , { 17464, 1843, }, }, - { " Hello world" , { 37727, 1843, }, }, - { "Hello World" , { 17464, 5427, }, }, - { " Hello World" , { 37727, 5427, }, }, - { " Hello World!" , { 37727, 5427, 0, }, }, - { "Hello, world!" , { 17464, 11, 1843, 0, }, }, - { " Hello, world!" , { 37727, 11, 1843, 0, }, }, - { " this is πŸ¦™.cpp" , { 437, 317, 12356, 99, 234, 13, 14743, }, }, - { "w048 7tuijk dsdfhu" , { 86, 15, 19, 23, 207, 22, 83, 3970, 27519, 26016, 3944, 14025, }, }, - { "Π½Π΅Ρ‰ΠΎ Π½Π° Π‘ΡŠΠ»Π³Π°Ρ€ΡΠΊΠΈ" , { 1603, 6476, 620, 91754, }, }, - { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰" , { 71374, 209, 71374, 114, 71374, 228, 155, 240, 220, 71374, 224, 155, 240, 211, 71374, 231, 71374, 115, 71374, 240, 155, 240, 210, 71374, 240, 71374, 95, 71374, 114, 71374, 214, 71374, 210, 71374, 236, 71374, 214, 155, 240, 210, 71374, 218, }, }, - { "πŸš€ (normal) πŸ˜Άβ€πŸŒ«οΈ (multiple emojis concatenated) βœ… (only emoji that has its own token)", { 10044, 95300, 334, 8754, 8, 33701, 114, 350, 222, 10044, 221, 104, 46713, 334, 34732, 996, 24250, 262, 80923, 8, 207, 37103, 214, 334, 5956, 89213, 344, 643, 895, 1377, 10728, 8, }, }, - { "Hello" , { 17464, }, }, - { " Hello" , { 37727, }, }, - { " Hello" , { 207, 37727, }, }, - { " Hello" , { 243, 37727, }, }, - { " Hello" , { 300, 37727, }, }, - { " Hello\n Hello" , { 300, 37727, 185, 300, 37727, }, }, - { "\n =" , { 185, 403, }, }, - { "' era" , { 6, 2906, }, }, - { "Hello, y'all! How are you 😁 ?ζˆ‘ζƒ³εœ¨appleε·₯作1314151倩～", { 17464, 11, 320, 6, 436, 0, 1724, 418, 340, 33701, 210, 3025, 19017, 612, 9407, 2681, 16, 18, 16, 19, 16, 20, 16, 1398, 68940, 239, }, }, - - }; - - return _k_tests; -} - -int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); - return 1; - } - - const std::string fname = argv[1]; - - std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; - } - - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); - - llama_model * model; - llama_context * ctx; - - llama_backend_init(); - - // load the vocab - { - auto mparams = llama_model_default_params(); - - mparams.vocab_only = true; - - model = llama_load_model_from_file(fname.c_str(), mparams); - - if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - return 1; - } - - auto cparams = llama_context_default_params(); - - ctx = llama_new_context_with_model(model, cparams); - - if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - llama_free_model(model); - return 1; - } - } - - if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) { - fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__); - llama_free_model(model); - llama_free(ctx); - return 2; - } - -#ifdef _WIN32 - // We need this for unicode console support - console::init(false, false); - atexit([]() { console::cleanup(); }); -#endif - - bool success = true; - - for (const auto & test_kv : k_tests()) { - const std::vector res = llama_tokenize(ctx, test_kv.first, false); - - printf("\n"); - printf("src: '%s'\n", test_kv.first.c_str()); - printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str()); - printf("tok: "); - for (const auto & tok : res) { - printf("%d ", tok); - } - printf("\n"); - - bool correct = res.size() == test_kv.second.size(); - for (int i = 0; i < (int) res.size() && correct; ++i) { - if (test_kv.second[i] != res[i]) { - correct = false; - } - } - - if (!correct) { - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); - fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, - llama_detokenize_bpe(ctx, res).c_str(), - llama_detokenize_bpe(ctx, test_kv.second).c_str()); - fprintf(stderr, "%s : expected tokens: ", __func__); - for (const auto & t : test_kv.second) { - fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); - } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : got tokens: ", __func__); - for (const auto & t : res) { - fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); - } - fprintf(stderr, "\n"); - } - } - - if (!fname_text.empty()) { - fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); - - std::string text; - { - std::ifstream ifs(fname_text); - if (!ifs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); - return 1; - } - text = std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator()); - } - - fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); - - const std::vector res = llama_tokenize(ctx, text, false); - - fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); - - { - const std::string fname_out = fname_text + ".tokcpp"; - - std::ofstream ofs(fname_out); - if (!ofs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); - return 1; - } - - for (const auto & tok : res) { - ofs << tok << " '" << llama_detokenize_bpe(ctx, std::vector{tok}) << "'" << std::endl; - } - } - - fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); - } - - llama_free_model(model); - llama_free(ctx); - - llama_backend_free(); - - return success ? 0 : 3; -} diff --git a/tests/test-tokenizer-0-falcon.cpp b/tests/test-tokenizer-0-falcon.cpp deleted file mode 100644 index 67f675a62..000000000 --- a/tests/test-tokenizer-0-falcon.cpp +++ /dev/null @@ -1,199 +0,0 @@ -#include "llama.h" -#include "common.h" -#include "console.h" - -#include -#include -#include -#include -#include - -// generate using test-tokenizer-0-falcon.py -static const std::map> & k_tests() { - static std::map> _k_tests = { - { "" , { }, }, - { " " , { 204, }, }, - { " " , { 258, }, }, - { " " , { 466, }, }, - { "\t" , { 192, }, }, - { "\n" , { 193, }, }, - { "\n\n" , { 1001, }, }, - { "\n\n\n" , { 11331, }, }, - { "\t\n" , { 19125, }, }, - { "Hello world" , { 9856, 1079, }, }, - { " Hello world" , { 23090, 1079, }, }, - { "Hello World" , { 9856, 2889, }, }, - { " Hello World" , { 23090, 2889, }, }, - { " Hello World!" , { 23090, 2889, 12, }, }, - { "Hello, world!" , { 9856, 23, 1079, 12, }, }, - { " Hello, world!" , { 23090, 23, 1079, 12, }, }, - { " this is πŸ¦™.cpp" , { 414, 304, 3346, 111, 231, 25, 29247, }, }, - { "w048 7tuijk dsdfhu" , { 98, 55866, 204, 34, 16682, 7149, 36190, 6869, 11481, }, }, - { "Π½Π΅Ρ‰ΠΎ Π½Π° Π‘ΡŠΠ»Π³Π°Ρ€ΡΠΊΠΈ" , { 150, 133, 6207, 151, 215, 150, 134, 5052, 133, 6279, 5052, 223, 151, 216, 49679, 123, 53110, 47043, 7795, }, }, - { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰" , { 38154, 206, 38154, 126, 38154, 225, 167, 237, 217, 38154, 221, 167, 237, 208, 38154, 228, 38154, 127, 38154, 237, 167, 237, 207, 38154, 237, 38154, 107, 38154, 126, 38154, 211, 38154, 207, 38154, 233, 38154, 211, 167, 237, 207, 38154, 215, }, }, - { "πŸš€ (normal) πŸ˜Άβ€πŸŒ«οΈ (multiple emojis concatenated) βœ… (only emoji that has its own token)", { 2571, 232, 206, 204, 19, 11003, 20, 8196, 126, 283, 219, 48778, 116, 13392, 204, 19, 51831, 732, 63209, 1741, 7955, 522, 20, 22438, 211, 204, 19, 7927, 53360, 325, 504, 701, 946, 10930, 20, }, }, - { "Hello" , { 9856, }, }, - { " Hello" , { 23090, }, }, - { " Hello" , { 204, 23090, }, }, - { " Hello" , { 258, 23090, }, }, - { " Hello" , { 466, 23090, }, }, - { " Hello\n Hello" , { 466, 23090, 742, 23090, }, }, - { " (" , { 204, 19, }, }, - { "\n =" , { 1212, 40, }, }, - { "' era" , { 18, 4932, }, }, - { "Hello, y'all! How are you 😁 ?ζˆ‘ζƒ³εœ¨appleε·₯作1314151倩～", { 9856, 23, 291, 18, 436, 12, 1265, 362, 299, 8196, 207, 204, 42, 50087, 123, 2727, 20300, 32022, 133, 234, 17419, 30137, 28, 7858, 181, 133, 236, }, }, - { "3" , { 30, }, }, - { "33" , { 3138, }, }, - { "333" , { 22287, }, }, - { "3333" , { 22287, 30, }, }, - { "33333" , { 22287, 3138, }, }, - { "333333" , { 22287, 22287, }, }, - { "3333333" , { 22287, 22287, 30, }, }, - { "33333333" , { 22287, 22287, 3138, }, }, - { "333333333" , { 22287, 22287, 22287, }, }, - }; - - return _k_tests; -} - -int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); - return 1; - } - - const std::string fname = argv[1]; - - std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; - } - - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); - - llama_model * model; - llama_context * ctx; - - llama_backend_init(); - - // load the vocab - { - auto mparams = llama_model_default_params(); - - mparams.vocab_only = true; - - model = llama_load_model_from_file(fname.c_str(), mparams); - - if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - return 1; - } - - auto cparams = llama_context_default_params(); - - ctx = llama_new_context_with_model(model, cparams); - - if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - llama_free_model(model); - return 1; - } - } - - if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) { - fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__); - llama_free_model(model); - llama_free(ctx); - return 2; - } - -#ifdef _WIN32 - // We need this for unicode console support - console::init(false, false); - atexit([]() { console::cleanup(); }); -#endif - - bool success = true; - - for (const auto & test_kv : k_tests()) { - const std::vector res = llama_tokenize(ctx, test_kv.first, false); - - printf("\n"); - printf("src: '%s'\n", test_kv.first.c_str()); - printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str()); - printf("tok: "); - for (const auto & tok : res) { - printf("%d ", tok); - } - printf("\n"); - - bool correct = res.size() == test_kv.second.size(); - for (int i = 0; i < (int) res.size() && correct; ++i) { - if (test_kv.second[i] != res[i]) { - correct = false; - } - } - - if (!correct) { - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); - fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, - llama_detokenize_bpe(ctx, res).c_str(), - llama_detokenize_bpe(ctx, test_kv.second).c_str()); - fprintf(stderr, "%s : expected tokens: ", __func__); - for (const auto & t : test_kv.second) { - fprintf(stderr, "%6d, ", t); - } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : got tokens: ", __func__); - for (const auto & t : res) { - fprintf(stderr, "%6d, ", t); - } - fprintf(stderr, "\n"); - - success = false; - } - } - - if (!fname_text.empty()) { - fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); - - std::string text; - { - std::ifstream ifs(fname_text); - if (!ifs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); - return 1; - } - text = std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator()); - } - - fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); - - const std::vector res = llama_tokenize(ctx, text, false); - - fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); - - { - const std::string fname_out = fname_text + ".tokcpp"; - - std::ofstream ofs(fname_out); - if (!ofs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); - return 1; - } - - for (const auto & tok : res) { - ofs << tok << " '" << llama_detokenize_bpe(ctx, std::vector{tok}) << "'" << std::endl; - } - } - - fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); - } - - llama_free_model(model); - llama_free(ctx); - - llama_backend_free(); - - return success ? 0 : 3; -} diff --git a/tests/test-tokenizer-0-llama-v3.cpp b/tests/test-tokenizer-0-llama-v3.cpp deleted file mode 100644 index ea9f5295f..000000000 --- a/tests/test-tokenizer-0-llama-v3.cpp +++ /dev/null @@ -1,199 +0,0 @@ -#include "llama.h" -#include "common.h" -#include "console.h" - -#include -#include -#include -#include -#include - -// generate using test-tokenizer-0-llama.py -static const std::map> & k_tests() { - static std::map> _k_tests = { - { "" , { }, }, - { " " , { 220, }, }, - { " " , { 256, }, }, - { " " , { 262, }, }, - { "\t" , { 197, }, }, - { "\n" , { 198, }, }, - { "\n\n" , { 271, }, }, - { "\n\n\n" , { 1432, }, }, - { "\t\n" , { 1602, }, }, - { "Hello world" , { 9906, 1917, }, }, - { " Hello world" , { 22691, 1917, }, }, - { "Hello World" , { 9906, 4435, }, }, - { " Hello World" , { 22691, 4435, }, }, - { " Hello World!" , { 22691, 4435, 0, }, }, - { "Hello, world!" , { 9906, 11, 1917, 0, }, }, - { " Hello, world!" , { 22691, 11, 1917, 0, }, }, - { " this is πŸ¦™.cpp" , { 420, 374, 11410, 99, 247, 13, 11055, }, }, - { "w048 7tuijk dsdfhu" , { 86, 23904, 220, 22, 83, 2005, 42908, 11729, 3013, 17156, }, }, - { "Π½Π΅Ρ‰ΠΎ Π½Π° Π‘ΡŠΠ»Π³Π°Ρ€ΡΠΊΠΈ" , { 79862, 102118, 13373, 64571, 34694, 3114, 112203, 80112, }, }, - { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰" , { 21549, 222, 98629, 241, 45358, 233, 21549, 237, 45358, 224, 21549, 244, 21549, 115, 21549, 253, 45358, 223, 21549, 253, 21549, 95, 98629, 227, 21549, 223, 21549, 249, 21549, 227, 45358, 223, 21549, 231, }, }, - { "πŸš€ (normal) πŸ˜Άβ€πŸŒ«οΈ (multiple emojis concatenated) βœ… (only emoji that has its own token)", { 9468, 248, 222, 320, 8416, 8, 27623, 114, 102470, 9468, 234, 104, 31643, 320, 36773, 100166, 98634, 8, 26602, 227, 320, 3323, 43465, 430, 706, 1202, 1866, 4037, 8, }, }, - { "Hello" , { 9906, }, }, - { " Hello" , { 22691, }, }, - { " Hello" , { 220, 22691, }, }, - { " Hello" , { 256, 22691, }, }, - { " Hello" , { 262, 22691, }, }, - { " Hello\n Hello" , { 262, 22691, 198, 262, 22691, }, }, - { " (" , { 320, }, }, - { "\n =" , { 198, 284, }, }, - { "' era" , { 6, 11639, }, }, - { "Hello, y'all! How are you 😁 ?ζˆ‘ζƒ³εœ¨appleε·₯作1314151倩～", { 9906, 11, 379, 65948, 0, 2650, 527, 499, 27623, 223, 949, 37046, 101067, 19000, 23182, 102301, 9263, 18136, 16, 36827, 21909, }, }, - { "3" , { 18, }, }, - { "33" , { 1644, }, }, - { "333" , { 8765, }, }, - { "3333" , { 8765, 18, }, }, - { "33333" , { 8765, 1644, }, }, - { "333333" , { 8765, 8765, }, }, - { "3333333" , { 8765, 8765, 18, }, }, - { "33333333" , { 8765, 8765, 1644, }, }, - { "333333333" , { 8765, 8765, 8765, }, }, - }; - - return _k_tests; -} - -int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); - return 1; - } - - const std::string fname = argv[1]; - - std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; - } - - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); - - llama_model * model; - llama_context * ctx; - - llama_backend_init(); - - // load the vocab - { - auto mparams = llama_model_default_params(); - - mparams.vocab_only = true; - - model = llama_load_model_from_file(fname.c_str(), mparams); - - if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - return 1; - } - - auto cparams = llama_context_default_params(); - - ctx = llama_new_context_with_model(model, cparams); - - if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - llama_free_model(model); - return 1; - } - } - - if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) { - fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__); - llama_free_model(model); - llama_free(ctx); - return 2; - } - -#ifdef _WIN32 - // We need this for unicode console support - console::init(false, false); - atexit([]() { console::cleanup(); }); -#endif - - bool success = true; - - for (const auto & test_kv : k_tests()) { - const std::vector res = llama_tokenize(ctx, test_kv.first, false); - - printf("\n"); - printf("src: '%s'\n", test_kv.first.c_str()); - printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str()); - printf("tok: "); - for (const auto & tok : res) { - printf("%d ", tok); - } - printf("\n"); - - bool correct = res.size() == test_kv.second.size(); - for (int i = 0; i < (int) res.size() && correct; ++i) { - if (test_kv.second[i] != res[i]) { - correct = false; - } - } - - if (!correct) { - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); - fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, - llama_detokenize_bpe(ctx, res).c_str(), - llama_detokenize_bpe(ctx, test_kv.second).c_str()); - fprintf(stderr, "%s : expected tokens: ", __func__); - for (const auto & t : test_kv.second) { - fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); - } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : got tokens: ", __func__); - for (const auto & t : res) { - fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); - } - fprintf(stderr, "\n"); - - success = false; - } - } - - if (!fname_text.empty()) { - fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); - - std::string text; - { - std::ifstream ifs(fname_text); - if (!ifs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); - return 1; - } - text = std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator()); - } - - fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); - - const std::vector res = llama_tokenize(ctx, text, false); - - fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); - - { - const std::string fname_out = fname_text + ".tokcpp"; - - std::ofstream ofs(fname_out); - if (!ofs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); - return 1; - } - - for (const auto & tok : res) { - ofs << tok << " '" << string_strip(llama_detokenize_bpe(ctx, std::vector{tok})) << "'" << std::endl; - } - } - - fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); - } - - llama_free_model(model); - llama_free(ctx); - - llama_backend_free(); - - return success ? 0 : 3; -} diff --git a/tests/test-tokenizer-0-llama.cpp b/tests/test-tokenizer-0-llama.cpp deleted file mode 100644 index f0634cfe5..000000000 --- a/tests/test-tokenizer-0-llama.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include "llama.h" -#include "common.h" -#include "console.h" - -#include -#include -#include -#include -#include - -// generate using test-tokenizer-0-llama.py -static const std::map> & k_tests() { - static std::map> _k_tests = { - { "" , { }, }, - { " " , { 259, }, }, - { " " , { 1678, }, }, - { " " , { 268, }, }, - { "\t" , { 29871, 12, }, }, - { "\n" , { 29871, 13, }, }, - { "\n\n" , { 29871, 13, 13, }, }, - { "\n\n\n" , { 29871, 13, 13, 13, }, }, - { "\t\n" , { 29871, 12, 13, }, }, - { "Hello world" , { 15043, 3186, }, }, - { " Hello world" , { 29871, 15043, 3186, }, }, - { "Hello World" , { 15043, 2787, }, }, - { " Hello World" , { 29871, 15043, 2787, }, }, - { " Hello World!" , { 29871, 15043, 2787, 29991, }, }, - { "Hello, world!" , { 15043, 29892, 3186, 29991, }, }, - { " Hello, world!" , { 29871, 15043, 29892, 3186, 29991, }, }, - { " this is πŸ¦™.cpp" , { 29871, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, - { "w048 7tuijk dsdfhu" , { 281, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, - { "Π½Π΅Ρ‰ΠΎ Π½Π° Π‘ΡŠΠ»Π³Π°Ρ€ΡΠΊΠΈ" , { 1538, 4851, 665, 1386, 29713, 1305, }, }, - { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰" , { 29871, 31849, 31324, 31934, 228, 162, 142, 228, 161, 146, 228, 162, 133, 228, 161, 153, 228, 161, 186, 31708, 228, 162, 132, 31708, 228, 161, 165, 31324, 228, 161, 136, 228, 161, 132, 228, 161, 158, 228, 161, 136, 228, 162, 132, 228, 161, 140, }, }, - { "πŸš€ (normal) πŸ˜Άβ€πŸŒ«οΈ (multiple emojis concatenated) βœ… (only emoji that has its own token)", { 29871, 243, 162, 157, 131, 313, 8945, 29897, 29871, 243, 162, 155, 185, 30722, 243, 162, 143, 174, 30598, 313, 20787, 953, 3848, 275, 16125, 630, 29897, 29871, 31681, 313, 6194, 953, 29877, 2397, 393, 756, 967, 1914, 5993, 29897, }, }, - { "Hello" , { 15043, }, }, - { " Hello" , { 29871, 15043, }, }, - { " Hello" , { 259, 15043, }, }, - { " Hello" , { 1678, 15043, }, }, - { " Hello" , { 268, 15043, }, }, - { " Hello\n Hello" , { 268, 15043, 13, 1678, 15043, }, }, - { " (" , { 29871, 313, }, }, - { "\n =" , { 29871, 13, 353, }, }, - { "' era" , { 525, 3152, }, }, - { "Hello, y'all! How are you 😁 ?ζˆ‘ζƒ³εœ¨appleε·₯作1314151倩～", { 15043, 29892, 343, 29915, 497, 29991, 1128, 526, 366, 29871, 243, 162, 155, 132, 1577, 30672, 31522, 30505, 11548, 31041, 30732, 29896, 29941, 29896, 29946, 29896, 29945, 29896, 30408, 30739, }, }, - { "3" , { 29871, 29941, }, }, - { "33" , { 29871, 29941, 29941, }, }, - { "333" , { 29871, 29941, 29941, 29941, }, }, - { "3333" , { 29871, 29941, 29941, 29941, 29941, }, }, - { "33333" , { 29871, 29941, 29941, 29941, 29941, 29941, }, }, - { "333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, }, }, - { "3333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, }, - { "33333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, }, - { "333333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, }, - }; - - return _k_tests; -} - -int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); - return 1; - } - - const std::string fname = argv[1]; - - std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; - } - - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); - - llama_model * model; - llama_context * ctx; - - llama_backend_init(); - - // load the vocab - { - auto mparams = llama_model_default_params(); - - mparams.vocab_only = true; - - model = llama_load_model_from_file(fname.c_str(), mparams); - - if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - return 1; - } - - auto cparams = llama_context_default_params(); - - ctx = llama_new_context_with_model(model, cparams); - - if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); - llama_free_model(model); - return 1; - } - } - - if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_SPM) { - fprintf(stderr, "%s : error: vocab type is not SPM\n", __func__); - llama_free_model(model); - llama_free(ctx); - return 2; - } - -#ifdef _WIN32 - // We need this for unicode console support - console::init(false, false); - atexit([]() { console::cleanup(); }); -#endif - - bool success = true; - - for (const auto & test_kv : k_tests()) { - const std::vector res_bos = llama_tokenize(ctx, test_kv.first, true); - const std::vector res_nobos = llama_tokenize(ctx, test_kv.first, false); - - printf("\n"); - printf("src: '%s'\n", test_kv.first.c_str()); - printf("res: '%s'\n", llama_detokenize_spm(ctx, res_bos).c_str()); - printf("tok: "); - for (const auto & tok : res_bos) { - printf("%d ", tok); - } - printf("\n"); - - bool correct = res_nobos.size() == test_kv.second.size() && res_bos.size() == res_nobos.size() + 1 && res_bos[0] == llama_token_bos(model); - - for (int i = 0; i < (int) res_nobos.size() && correct; ++i) { - if (test_kv.second[i] != res_bos[i + 1]) { - correct = false; - } - if (test_kv.second[i] != res_nobos[i]) { - correct = false; - } - } - - if (!correct) { - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); - fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, - llama_detokenize_spm(ctx, res_nobos).c_str(), - llama_detokenize_spm(ctx, test_kv.second).c_str()); - fprintf(stderr, "%s : expected tokens: ", __func__); - for (const auto & t : test_kv.second) { - fprintf(stderr, "%6d, ", t); - } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : got tokens: ", __func__); - for (const auto & t : res_nobos) { - fprintf(stderr, "%6d, ", t); - } - fprintf(stderr, "\n"); - - success = false; - } - } - - if (!fname_text.empty()) { - fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); - - std::string text; - { - std::ifstream ifs(fname_text); - if (!ifs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); - return 1; - } - text = std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator()); - } - - fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); - - const std::vector res = llama_tokenize(ctx, text, true); - - fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); - - { - const std::string fname_out = fname_text + ".tokcpp"; - - std::ofstream ofs(fname_out); - if (!ofs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); - return 1; - } - - for (const auto & tok : res) { - ofs << tok << " '" << llama_detokenize_spm(ctx, std::vector{tok}) << "'" << std::endl; - } - } - - fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); - } - - llama_free_model(model); - llama_free(ctx); - - llama_backend_free(); - - return success ? 0 : 3; -} diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp new file mode 100644 index 000000000..bb00241f8 --- /dev/null +++ b/tests/test-tokenizer-0.cpp @@ -0,0 +1,263 @@ +#include "llama.h" +#include "common.h" +#include "console.h" + +#include +#include +#include +#include +#include + +//static const std::map> & k_tests() { +// static std::map> _k_tests = { +// { "" , { }, }, +// { " " , { 220, }, }, +// { " " , { 256, }, }, +// { " " , { 262, }, }, +// { "\t" , { 197, }, }, +// { "\n" , { 198, }, }, +// { "\n\n" , { 271, }, }, +// { "\n\n\n" , { 1432, }, }, +// { "\t\n" , { 1602, }, }, +// { "Hello world" , { 9906, 1917, }, }, +// { " Hello world" , { 22691, 1917, }, }, +// { "Hello World" , { 9906, 4435, }, }, +// { " Hello World" , { 22691, 4435, }, }, +// { " Hello World!" , { 22691, 4435, 0, }, }, +// { "Hello, world!" , { 9906, 11, 1917, 0, }, }, +// { " Hello, world!" , { 22691, 11, 1917, 0, }, }, +// { " this is πŸ¦™.cpp" , { 420, 374, 11410, 99, 247, 13, 11055, }, }, +// { "w048 7tuijk dsdfhu" , { 86, 23904, 220, 22, 83, 2005, 42908, 11729, 3013, 17156, }, }, +// { "Π½Π΅Ρ‰ΠΎ Π½Π° Π‘ΡŠΠ»Π³Π°Ρ€ΡΠΊΠΈ" , { 79862, 102118, 13373, 64571, 34694, 3114, 112203, 80112, }, }, +// { "αž€αžΆαž“αŸ‹αžαŸ‚αž–αž·αžŸαŸαžŸαž’αžΆαž…αžαž›αž…αŸαž‰" , { 21549, 222, 98629, 241, 45358, 233, 21549, 237, 45358, 224, 21549, 244, 21549, 115, 21549, 253, 45358, 223, 21549, 253, 21549, 95, 98629, 227, 21549, 223, 21549, 249, 21549, 227, 45358, 223, 21549, 231, }, }, +// { "πŸš€ (normal) πŸ˜Άβ€πŸŒ«οΈ (multiple emojis concatenated) βœ… (only emoji that has its own token)", { 9468, 248, 222, 320, 8416, 8, 27623, 114, 102470, 9468, 234, 104, 31643, 320, 36773, 100166, 98634, 8, 26602, 227, 320, 3323, 43465, 430, 706, 1202, 1866, 4037, 8, }, }, +// { "Hello" , { 9906, }, }, +// { " Hello" , { 22691, }, }, +// { " Hello" , { 220, 22691, }, }, +// { " Hello" , { 256, 22691, }, }, +// { " Hello" , { 262, 22691, }, }, +// { " Hello\n Hello" , { 262, 22691, 198, 262, 22691, }, }, +// { " (" , { 320, }, }, +// { "\n =" , { 198, 284, }, }, +// { "' era" , { 6, 11639, }, }, +// { "Hello, y'all! How are you 😁 ?ζˆ‘ζƒ³εœ¨appleε·₯作1314151倩～", { 9906, 11, 379, 65948, 0, 2650, 527, 499, 27623, 223, 949, 37046, 101067, 19000, 23182, 102301, 9263, 18136, 16, 36827, 21909, }, }, +// { "3" , { 18, }, }, +// { "33" , { 1644, }, }, +// { "333" , { 8765, }, }, +// { "3333" , { 8765, 18, }, }, +// { "33333" , { 8765, 1644, }, }, +// { "333333" , { 8765, 8765, }, }, +// { "3333333" , { 8765, 8765, 18, }, }, +// { "33333333" , { 8765, 8765, 1644, }, }, +// { "333333333" , { 8765, 8765, 8765, }, }, +// }; +// +// return _k_tests; +//} + +static std::map> read_tests(const std::string & fname_inp, const std::string & fname_out) { + std::map> tests; + + std::ifstream ifs_inp(fname_inp); + if (!ifs_inp) { + fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_inp.c_str()); + return tests; + } + + std::string sraw((std::istreambuf_iterator(ifs_inp)), std::istreambuf_iterator()); + + std::ifstream ifs_out(fname_out); + if (!ifs_out) { + fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); + return tests; + } + + std::vector sout; + for (std::string line; std::getline(ifs_out, line);) { + sout.push_back(line); + } + + const std::string sep = "\n__ggml_vocab_test__\n"; + + std::vector sinp; + + size_t pos = 0; + while (pos < sraw.size()) { + const size_t next = sraw.find(sep, pos); + if (next == std::string::npos) { + sinp.push_back(sraw.substr(pos)); + break; + } + sinp.push_back(sraw.substr(pos, next - pos)); + pos = next + sep.size(); + } + + if (sinp.size() != sout.size()) { + fprintf(stderr, "%s : error: input and output files have different number of tests\n", __func__); + return tests; + } + + for (size_t i = 0; i < sinp.size(); ++i) { + const std::string & s = sinp[i]; + const std::string & o = string_strip(sout[i]); + + std::vector toks; + + size_t pos = 0; + while (pos < o.size()) { + size_t next = o.find(' ', pos); + if (next == std::string::npos) { + next = o.size(); + } + const std::string stok = o.substr(pos, next - pos); + toks.push_back(std::stoi(stok)); + pos = next + 1; + } + + tests[s] = toks; + } + + return tests; +} + +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); + return 1; + } + + const std::string fname = argv[1]; + + const std::string fname_inp = fname + ".inp"; + const std::string fname_out = fname + ".out"; + + std::string fname_text; + if (argc > 2) { + fname_text = argv[2]; + } + + fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); + + llama_model * model; + llama_context * ctx; + + llama_backend_init(); + + // load the vocab + { + auto mparams = llama_model_default_params(); + + mparams.vocab_only = true; + + model = llama_load_model_from_file(fname.c_str(), mparams); + + if (model == NULL) { + fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + return 1; + } + + auto cparams = llama_context_default_params(); + + ctx = llama_new_context_with_model(model, cparams); + + if (ctx == NULL) { + fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + llama_free_model(model); + return 1; + } + } + +#ifdef _WIN32 + // We need this for unicode console support + console::init(false, false); + atexit([]() { console::cleanup(); }); +#endif + + bool success = true; + + const auto k_tests = read_tests(fname_inp, fname_out); + + const bool add_special = false; + + for (const auto & test_kv : k_tests) { + const std::vector res = llama_tokenize(ctx, test_kv.first, add_special); + + printf("\n"); + printf("src: '%s'\n", test_kv.first.c_str()); + printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str()); + printf("tok: "); + for (const auto & tok : res) { + printf("%d ", tok); + } + printf("\n"); + + bool correct = res.size() == test_kv.second.size(); + for (int i = 0; i < (int) res.size() && correct; ++i) { + if (test_kv.second[i] != res[i]) { + correct = false; + } + } + + if (!correct) { + fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); + fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, + llama_detokenize_bpe(ctx, res).c_str(), + llama_detokenize_bpe(ctx, test_kv.second).c_str()); + fprintf(stderr, "%s : expected tokens: ", __func__); + for (const auto & t : test_kv.second) { + fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); + } + fprintf(stderr, "\n"); + fprintf(stderr, "%s : got tokens: ", __func__); + for (const auto & t : res) { + fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str()); + } + fprintf(stderr, "\n"); + + success = false; + } + } + + if (!fname_text.empty()) { + fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); + + std::string text; + { + std::ifstream ifs(fname_text); + if (!ifs) { + fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); + return 1; + } + text = std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator()); + } + + fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); + + const std::vector res = llama_tokenize(ctx, text, add_special); + + fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); + + { + const std::string fname_out = fname_text + ".tokcpp"; + + std::ofstream ofs(fname_out); + if (!ofs) { + fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); + return 1; + } + + for (const auto & tok : res) { + ofs << tok << " '" << string_strip(llama_detokenize_bpe(ctx, std::vector{tok})) << "'" << std::endl; + } + } + + fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); + } + + llama_free_model(model); + llama_free(ctx); + + llama_backend_free(); + + return success ? 0 : 3; +} diff --git a/tests/test-tokenizer-1-llama.cpp b/tests/test-tokenizer-1-spm.cpp similarity index 100% rename from tests/test-tokenizer-1-llama.cpp rename to tests/test-tokenizer-1-spm.cpp