tests : refactor vocab tests
ggml-ci
This commit is contained in:
Georgi Gerganov
parent
ef4cca9e87
commit
43708d22c3
15 changed files with 316 additions and 1010 deletions
|
|
@ -46,8 +46,8 @@ else:
|
||||||
|
|
||||||
# TODO: add models here, base models preferred
|
# TODO: add models here, base models preferred
|
||||||
models = [
|
models = [
|
||||||
{ "name": "llama-v2", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
|
{ "name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },
|
||||||
{ "name": "llama-v3", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
|
{ "name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },
|
||||||
{ "name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
|
{ "name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },
|
||||||
{ "name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
|
{ "name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },
|
||||||
{ "name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
|
{ "name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },
|
||||||
|
|
@ -64,7 +64,7 @@ def download_file_with_auth(url, token, save_path):
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
with open(save_path, 'wb') as f:
|
with open(save_path, 'wb') as f:
|
||||||
f.write(response.content)
|
f.write(response.content)
|
||||||
print("File downloaded successfully.")
|
print(f"File {save_path} downloaded successfully")
|
||||||
else:
|
else:
|
||||||
print(f"Failed to download file. Status code: {response.status_code}")
|
print(f"Failed to download file. Status code: {response.status_code}")
|
||||||
|
|
||||||
|
|
@ -82,6 +82,10 @@ for model in models:
|
||||||
|
|
||||||
print(f"Downloading {name} to models/tokenizers/{name}")
|
print(f"Downloading {name} to models/tokenizers/{name}")
|
||||||
|
|
||||||
|
url = f"{repo}/raw/main/config.json"
|
||||||
|
save_path = f"models/tokenizers/{name}/config.json"
|
||||||
|
download_file_with_auth(url, token, save_path)
|
||||||
|
|
||||||
url = f"{repo}/raw/main/tokenizer.json"
|
url = f"{repo}/raw/main/tokenizer.json"
|
||||||
save_path = f"models/tokenizers/{name}/tokenizer.json"
|
save_path = f"models/tokenizers/{name}/tokenizer.json"
|
||||||
download_file_with_auth(url, token, save_path)
|
download_file_with_auth(url, token, save_path)
|
||||||
|
|
@ -219,7 +223,7 @@ tests = [
|
||||||
"333333333",
|
"333333333",
|
||||||
]
|
]
|
||||||
|
|
||||||
# write the tests in ./models/test-vocab-inp.txt
|
# write the tests to ./models/ggml-vocab-{name}.gguf.inp
|
||||||
# the format is:
|
# the format is:
|
||||||
#
|
#
|
||||||
# test0
|
# test0
|
||||||
|
|
@ -229,14 +233,7 @@ tests = [
|
||||||
# ...
|
# ...
|
||||||
#
|
#
|
||||||
|
|
||||||
with open(f"models/test-vocab-inp.txt", "w") as f:
|
# with each model, encode all tests and write the results in ./models/ggml-vocab-{name}.gguf.out
|
||||||
for text in tests:
|
|
||||||
f.write(f"{text}")
|
|
||||||
f.write("\n__ggml_vocab_test__\n")
|
|
||||||
|
|
||||||
print("Tests written in ./models/test-vocab-inp.txt")
|
|
||||||
|
|
||||||
# with each model, encode all tests and write the results in ./models/test-vocab-out-{name}.txt
|
|
||||||
# for each test, write the resulting tokens on a separate line
|
# for each test, write the resulting tokens on a separate line
|
||||||
|
|
||||||
for model in models:
|
for model in models:
|
||||||
|
|
@ -247,11 +244,27 @@ for model in models:
|
||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
|
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
|
||||||
|
|
||||||
with open(f"models/test-vocab-out-{name}.txt", "w") as f:
|
with open(f"models/ggml-vocab-{name}.gguf.inp", "w") as f:
|
||||||
for text in tests:
|
for text in tests:
|
||||||
res = tokenizer.encode(text)
|
f.write(f"{text}")
|
||||||
|
f.write("\n__ggml_vocab_test__\n")
|
||||||
|
|
||||||
|
with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:
|
||||||
|
for text in tests:
|
||||||
|
res = tokenizer.encode(text, add_special_tokens=False)
|
||||||
for r in res:
|
for r in res:
|
||||||
f.write(f" {r}")
|
f.write(f" {r}")
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
print(f"Test results for {name} written in ./models/test-vocab-out-{name}.txt")
|
print(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")
|
||||||
|
|
||||||
|
# generate commands for creating vocab files
|
||||||
|
|
||||||
|
print("\nRun the following commands to generate the vocab files for testing:\n")
|
||||||
|
|
||||||
|
for model in models:
|
||||||
|
name = model["name"]
|
||||||
|
|
||||||
|
print(f"python3 convert-hf-to-gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only")
|
||||||
|
|
||||||
|
print("\n")
|
||||||
|
|
|
||||||
|
|
@ -283,7 +283,7 @@ class Model(ABC):
|
||||||
# don't do this manually - use the convert-hf-to-gguf-update.py script!
|
# don't do this manually - use the convert-hf-to-gguf-update.py script!
|
||||||
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
|
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
|
||||||
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
|
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
|
||||||
res = "llama-v3"
|
res = "llama-bpe"
|
||||||
if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754":
|
if chkhsh == "049ecf7629871e3041641907f3de7c733e4dbfdc736f57d882ba0b0845599754":
|
||||||
# ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base
|
# ref: https://huggingface.co/deepseek-ai/deepseek-llm-7b-base
|
||||||
res = "deepseek-llm"
|
res = "deepseek-llm"
|
||||||
|
|
|
||||||
|
|
@ -4339,8 +4339,9 @@ static void llm_load_vocab(
|
||||||
tokenizer_pre == "default") {
|
tokenizer_pre == "default") {
|
||||||
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
|
||||||
} else if (
|
} else if (
|
||||||
tokenizer_pre == "llama3" ||
|
tokenizer_pre == "llama3" ||
|
||||||
tokenizer_pre == "llama-v3") {
|
tokenizer_pre == "llama-v3" ||
|
||||||
|
tokenizer_pre == "llama-bpe") {
|
||||||
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
|
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
|
||||||
} else if (
|
} else if (
|
||||||
tokenizer_pre == "deepseek-llm") {
|
tokenizer_pre == "deepseek-llm") {
|
||||||
|
|
@ -12583,7 +12584,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
|
||||||
} break;
|
} break;
|
||||||
case LLAMA_VOCAB_TYPE_BPE:
|
case LLAMA_VOCAB_TYPE_BPE:
|
||||||
{
|
{
|
||||||
if (add_special && vocab.special_add_bos == 1) {
|
if (add_special && vocab.special_add_bos != 0) {
|
||||||
GGML_ASSERT(vocab.special_bos_id != -1);
|
GGML_ASSERT(vocab.special_bos_id != -1);
|
||||||
output.push_back(vocab.special_bos_id);
|
output.push_back(vocab.special_bos_id);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -65,21 +65,16 @@ function(llama_target_and_test source)
|
||||||
set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL})
|
set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL})
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
# llama_target_and_test(test-double-float.cpp) # SLOW
|
# build test-tokenizer-0 target once and add many tests
|
||||||
llama_target_and_test(test-quantize-fns.cpp)
|
add_executable(test-tokenizer-0 test-tokenizer-0.cpp get-model.cpp)
|
||||||
llama_target_and_test(test-quantize-perf.cpp)
|
target_link_libraries(test-tokenizer-0 PRIVATE common)
|
||||||
llama_target_and_test(test-sampling.cpp)
|
install(TARGETS test-tokenizer-0 RUNTIME)
|
||||||
llama_target_and_test(test-chat-template.cpp)
|
|
||||||
|
|
||||||
llama_target_and_test(test-tokenizer-0-llama.cpp NAME test-tokenizer-0-llama ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf)
|
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-spm.gguf)
|
||||||
llama_target_and_test(test-tokenizer-0-llama-v3.cpp NAME test-tokenizer-0-llama-v3 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-v3.gguf)
|
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-bpe.gguf)
|
||||||
llama_target_and_test(test-tokenizer-0-falcon.cpp NAME test-tokenizer-0-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf)
|
llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-falcon.gguf)
|
||||||
|
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-llm.gguf)
|
||||||
llama_target_and_test(test-tokenizer-0-deepseek-coder.cpp NAME test-tokenizer-0-deepseek-coder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-coder.gguf)
|
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-coder.gguf)
|
||||||
llama_target_and_test(test-tokenizer-0-deepseek-llm.cpp NAME test-tokenizer-0-deepseek-llm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-deepseek-llm.gguf)
|
|
||||||
|
|
||||||
llama_target_and_test(test-tokenizer-1-llama.cpp NAME test-tokenizer-1-llama ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama.gguf)
|
|
||||||
llama_target_and_test(test-tokenizer-1-llama.cpp NAME test-tokenizer-1-baichuan ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-baichuan.gguf)
|
|
||||||
|
|
||||||
# build test-tokenizer-1-bpe target once and add many tests
|
# build test-tokenizer-1-bpe target once and add many tests
|
||||||
add_executable(test-tokenizer-1-bpe test-tokenizer-1-bpe.cpp get-model.cpp)
|
add_executable(test-tokenizer-1-bpe test-tokenizer-1-bpe.cpp get-model.cpp)
|
||||||
|
|
@ -96,9 +91,19 @@ llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-starcoder ARGS ${CMAKE_CUR
|
||||||
llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt2 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt2.gguf)
|
llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt2 ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-gpt2.gguf)
|
||||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-bloom ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-bloom.gguf) # BIG
|
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-bloom ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-bloom.gguf) # BIG
|
||||||
|
|
||||||
|
# build test-tokenizer-1-spm target once and add many tests
|
||||||
|
add_executable(test-tokenizer-1-spm test-tokenizer-1-spm.cpp get-model.cpp)
|
||||||
|
target_link_libraries(test-tokenizer-1-spm PRIVATE common)
|
||||||
|
install(TARGETS test-tokenizer-1-spm RUNTIME)
|
||||||
|
|
||||||
|
llama_target_and_test(test-tokenizer-1-spm NAME test-tokenizer-1-llama-spm ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-llama-spm.gguf)
|
||||||
|
llama_target_and_test(test-tokenizer-1-spm NAME test-tokenizer-1-baichuan ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../models/ggml-vocab-baichuan.gguf)
|
||||||
|
|
||||||
|
# llama_target_and_test(test-double-float.cpp) # SLOW
|
||||||
|
llama_target_and_test(test-quantize-fns.cpp)
|
||||||
|
llama_target_and_test(test-quantize-perf.cpp)
|
||||||
|
llama_target_and_test(test-sampling.cpp)
|
||||||
|
llama_target_and_test(test-chat-template.cpp)
|
||||||
|
|
||||||
llama_target_and_test(test-grammar-parser.cpp)
|
llama_target_and_test(test-grammar-parser.cpp)
|
||||||
llama_target_and_test(test-llama-grammar.cpp)
|
llama_target_and_test(test-llama-grammar.cpp)
|
||||||
|
|
|
||||||
|
|
@ -1,188 +0,0 @@
|
||||||
#include "llama.h"
|
|
||||||
#include "common.h"
|
|
||||||
#include "console.h"
|
|
||||||
|
|
||||||
#include <cstdio>
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
|
||||||
#include <vector>
|
|
||||||
#include <fstream>
|
|
||||||
|
|
||||||
// generate using test-tokenizer-0-falcon.py
|
|
||||||
static const std::map<std::string, std::vector<llama_token>> & k_tests() {
|
|
||||||
static std::map<std::string, std::vector<llama_token>> _k_tests = {
|
|
||||||
{ "" , { }, },
|
|
||||||
{ " " , { 207, }, },
|
|
||||||
{ " " , { 243, }, },
|
|
||||||
{ " " , { 315, }, },
|
|
||||||
{ "\t" , { 184, }, },
|
|
||||||
{ "\n" , { 185, }, },
|
|
||||||
{ "\t\n" , { 184, 185, }, },
|
|
||||||
{ "Hello world" , { 17535, 1835, }, },
|
|
||||||
{ " Hello world" , { 414, 9489, 1835, }, },
|
|
||||||
{ "Hello World" , { 17535, 5414, }, },
|
|
||||||
{ " Hello World" , { 414, 9489, 5414, }, },
|
|
||||||
{ " Hello World!" , { 414, 9489, 5414, 0, }, },
|
|
||||||
{ "Hello, world!" , { 17535, 11, 1835, 0, }, },
|
|
||||||
{ " Hello, world!" , { 414, 9489, 11, 1835, 0, }, },
|
|
||||||
{ " this is 🦙.cpp" , { 437, 317, 12394, 99, 234, 13, 14789, }, },
|
|
||||||
{ "w048 7tuijk dsdfhu" , { 86, 15, 19, 23, 207, 22, 83, 3963, 27659, 26078, 3934, 14072, }, },
|
|
||||||
{ "нещо на Български" , { 1593, 6478, 616, 2251, 14994, }, },
|
|
||||||
{ "កាន់តែពិសេសអាចខលចេញ" , { 155, 239, 209, 155, 239, 114, 155, 239, 228, 155, 240, 220, 155, 239, 224, 155, 240, 211, 155, 239, 231, 155, 239, 115, 155, 239, 240, 155, 240, 210, 155, 239, 240, 155, 239, 95, 155, 239, 114, 155, 239, 214, 155, 239, 210, 155, 239, 236, 155, 239, 214, 155, 240, 210, 155, 239, 218, }, },
|
|
||||||
{ "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)", { 10047, 235, 209, 334, 8760, 8, 12394, 233, 114, 350, 222, 10047, 221, 104, 169, 116, 224, 334, 4684, 3909, 992, 24330, 262, 29651, 612, 8, 207, 156, 237, 214, 334, 5950, 992, 78, 12896, 344, 638, 891, 1372, 10736, 8, }, },
|
|
||||||
{ "Hello" , { 17535, }, },
|
|
||||||
{ " Hello" , { 414, 9489, }, },
|
|
||||||
{ " Hello" , { 207, 414, 9489, }, },
|
|
||||||
{ " Hello" , { 243, 414, 9489, }, },
|
|
||||||
{ " Hello" , { 315, 414, 9489, }, },
|
|
||||||
{ " Hello\n Hello" , { 315, 414, 9489, 185, 315, 414, 9489, }, },
|
|
||||||
{ "\n =" , { 185, 405, }, },
|
|
||||||
{ "' era" , { 6, 2895, }, },
|
|
||||||
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 17535, 11, 320, 6, 435, 0, 1717, 417, 340, 12394, 233, 210, 3015, 19100, 608, 9413, 2668, 16, 18, 16, 19, 16, 20, 16, 1393, 169, 121, 239, }, },
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
return _k_tests;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
|
||||||
if (argc < 2) {
|
|
||||||
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string fname = argv[1];
|
|
||||||
|
|
||||||
std::string fname_text;
|
|
||||||
if (argc > 2) {
|
|
||||||
fname_text = argv[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
|
||||||
|
|
||||||
llama_model * model;
|
|
||||||
llama_context * ctx;
|
|
||||||
|
|
||||||
llama_backend_init();
|
|
||||||
|
|
||||||
// load the vocab
|
|
||||||
{
|
|
||||||
auto mparams = llama_model_default_params();
|
|
||||||
|
|
||||||
mparams.vocab_only = true;
|
|
||||||
|
|
||||||
model = llama_load_model_from_file(fname.c_str(), mparams);
|
|
||||||
|
|
||||||
if (model == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto cparams = llama_context_default_params();
|
|
||||||
|
|
||||||
ctx = llama_new_context_with_model(model, cparams);
|
|
||||||
|
|
||||||
if (ctx == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
llama_free_model(model);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) {
|
|
||||||
fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__);
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
return 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef _WIN32
|
|
||||||
// We need this for unicode console support
|
|
||||||
console::init(false, false);
|
|
||||||
atexit([]() { console::cleanup(); });
|
|
||||||
#endif
|
|
||||||
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
for (const auto & test_kv : k_tests()) {
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, false);
|
|
||||||
|
|
||||||
printf("\n");
|
|
||||||
printf("src: '%s'\n", test_kv.first.c_str());
|
|
||||||
printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str());
|
|
||||||
printf("tok: ");
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
printf("%d ", tok);
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
|
|
||||||
bool correct = res.size() == test_kv.second.size();
|
|
||||||
for (int i = 0; i < (int) res.size() && correct; ++i) {
|
|
||||||
if (test_kv.second[i] != res[i]) {
|
|
||||||
correct = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!correct) {
|
|
||||||
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
|
|
||||||
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
|
|
||||||
llama_detokenize_bpe(ctx, res).c_str(),
|
|
||||||
llama_detokenize_bpe(ctx, test_kv.second).c_str());
|
|
||||||
fprintf(stderr, "%s : expected tokens: ", __func__);
|
|
||||||
for (const auto & t : test_kv.second) {
|
|
||||||
fprintf(stderr, "%6d, ", t);
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
fprintf(stderr, "%s : got tokens: ", __func__);
|
|
||||||
for (const auto & t : res) {
|
|
||||||
fprintf(stderr, "%6d, ", t);
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
|
|
||||||
success = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!fname_text.empty()) {
|
|
||||||
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
|
|
||||||
|
|
||||||
std::string text;
|
|
||||||
{
|
|
||||||
std::ifstream ifs(fname_text);
|
|
||||||
if (!ifs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
|
|
||||||
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, text, false);
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
|
|
||||||
|
|
||||||
{
|
|
||||||
const std::string fname_out = fname_text + ".tokcpp";
|
|
||||||
|
|
||||||
std::ofstream ofs(fname_out);
|
|
||||||
if (!ofs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
ofs << tok << " '" << llama_detokenize_bpe(ctx, std::vector<int>{tok}) << "'" << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
|
|
||||||
}
|
|
||||||
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
|
|
||||||
llama_backend_free();
|
|
||||||
|
|
||||||
return success ? 0 : 3;
|
|
||||||
}
|
|
||||||
|
|
@ -1,186 +0,0 @@
|
||||||
#include "llama.h"
|
|
||||||
#include "common.h"
|
|
||||||
#include "console.h"
|
|
||||||
|
|
||||||
#include <cstdio>
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
|
||||||
#include <vector>
|
|
||||||
#include <fstream>
|
|
||||||
|
|
||||||
// generate using test-tokenizer-0-falcon.py
|
|
||||||
static const std::map<std::string, std::vector<llama_token>> & k_tests() {
|
|
||||||
static std::map<std::string, std::vector<llama_token>> _k_tests = {
|
|
||||||
{ "" , { }, },
|
|
||||||
{ " " , { 207, }, },
|
|
||||||
{ " " , { 243, }, },
|
|
||||||
{ " " , { 300, }, },
|
|
||||||
{ "\t" , { 184, }, },
|
|
||||||
{ "\n" , { 185, }, },
|
|
||||||
{ "\t\n" , { 184, 185, }, },
|
|
||||||
{ "Hello world" , { 17464, 1843, }, },
|
|
||||||
{ " Hello world" , { 37727, 1843, }, },
|
|
||||||
{ "Hello World" , { 17464, 5427, }, },
|
|
||||||
{ " Hello World" , { 37727, 5427, }, },
|
|
||||||
{ " Hello World!" , { 37727, 5427, 0, }, },
|
|
||||||
{ "Hello, world!" , { 17464, 11, 1843, 0, }, },
|
|
||||||
{ " Hello, world!" , { 37727, 11, 1843, 0, }, },
|
|
||||||
{ " this is 🦙.cpp" , { 437, 317, 12356, 99, 234, 13, 14743, }, },
|
|
||||||
{ "w048 7tuijk dsdfhu" , { 86, 15, 19, 23, 207, 22, 83, 3970, 27519, 26016, 3944, 14025, }, },
|
|
||||||
{ "нещо на Български" , { 1603, 6476, 620, 91754, }, },
|
|
||||||
{ "កាន់តែពិសេសអាចខលចេញ" , { 71374, 209, 71374, 114, 71374, 228, 155, 240, 220, 71374, 224, 155, 240, 211, 71374, 231, 71374, 115, 71374, 240, 155, 240, 210, 71374, 240, 71374, 95, 71374, 114, 71374, 214, 71374, 210, 71374, 236, 71374, 214, 155, 240, 210, 71374, 218, }, },
|
|
||||||
{ "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)", { 10044, 95300, 334, 8754, 8, 33701, 114, 350, 222, 10044, 221, 104, 46713, 334, 34732, 996, 24250, 262, 80923, 8, 207, 37103, 214, 334, 5956, 89213, 344, 643, 895, 1377, 10728, 8, }, },
|
|
||||||
{ "Hello" , { 17464, }, },
|
|
||||||
{ " Hello" , { 37727, }, },
|
|
||||||
{ " Hello" , { 207, 37727, }, },
|
|
||||||
{ " Hello" , { 243, 37727, }, },
|
|
||||||
{ " Hello" , { 300, 37727, }, },
|
|
||||||
{ " Hello\n Hello" , { 300, 37727, 185, 300, 37727, }, },
|
|
||||||
{ "\n =" , { 185, 403, }, },
|
|
||||||
{ "' era" , { 6, 2906, }, },
|
|
||||||
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 17464, 11, 320, 6, 436, 0, 1724, 418, 340, 33701, 210, 3025, 19017, 612, 9407, 2681, 16, 18, 16, 19, 16, 20, 16, 1398, 68940, 239, }, },
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
return _k_tests;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
|
||||||
if (argc < 2) {
|
|
||||||
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string fname = argv[1];
|
|
||||||
|
|
||||||
std::string fname_text;
|
|
||||||
if (argc > 2) {
|
|
||||||
fname_text = argv[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
|
||||||
|
|
||||||
llama_model * model;
|
|
||||||
llama_context * ctx;
|
|
||||||
|
|
||||||
llama_backend_init();
|
|
||||||
|
|
||||||
// load the vocab
|
|
||||||
{
|
|
||||||
auto mparams = llama_model_default_params();
|
|
||||||
|
|
||||||
mparams.vocab_only = true;
|
|
||||||
|
|
||||||
model = llama_load_model_from_file(fname.c_str(), mparams);
|
|
||||||
|
|
||||||
if (model == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto cparams = llama_context_default_params();
|
|
||||||
|
|
||||||
ctx = llama_new_context_with_model(model, cparams);
|
|
||||||
|
|
||||||
if (ctx == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
llama_free_model(model);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) {
|
|
||||||
fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__);
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
return 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef _WIN32
|
|
||||||
// We need this for unicode console support
|
|
||||||
console::init(false, false);
|
|
||||||
atexit([]() { console::cleanup(); });
|
|
||||||
#endif
|
|
||||||
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
for (const auto & test_kv : k_tests()) {
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, false);
|
|
||||||
|
|
||||||
printf("\n");
|
|
||||||
printf("src: '%s'\n", test_kv.first.c_str());
|
|
||||||
printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str());
|
|
||||||
printf("tok: ");
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
printf("%d ", tok);
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
|
|
||||||
bool correct = res.size() == test_kv.second.size();
|
|
||||||
for (int i = 0; i < (int) res.size() && correct; ++i) {
|
|
||||||
if (test_kv.second[i] != res[i]) {
|
|
||||||
correct = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!correct) {
|
|
||||||
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
|
|
||||||
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
|
|
||||||
llama_detokenize_bpe(ctx, res).c_str(),
|
|
||||||
llama_detokenize_bpe(ctx, test_kv.second).c_str());
|
|
||||||
fprintf(stderr, "%s : expected tokens: ", __func__);
|
|
||||||
for (const auto & t : test_kv.second) {
|
|
||||||
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
fprintf(stderr, "%s : got tokens: ", __func__);
|
|
||||||
for (const auto & t : res) {
|
|
||||||
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!fname_text.empty()) {
|
|
||||||
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
|
|
||||||
|
|
||||||
std::string text;
|
|
||||||
{
|
|
||||||
std::ifstream ifs(fname_text);
|
|
||||||
if (!ifs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
|
|
||||||
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, text, false);
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
|
|
||||||
|
|
||||||
{
|
|
||||||
const std::string fname_out = fname_text + ".tokcpp";
|
|
||||||
|
|
||||||
std::ofstream ofs(fname_out);
|
|
||||||
if (!ofs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
ofs << tok << " '" << llama_detokenize_bpe(ctx, std::vector<int>{tok}) << "'" << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
|
|
||||||
}
|
|
||||||
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
|
|
||||||
llama_backend_free();
|
|
||||||
|
|
||||||
return success ? 0 : 3;
|
|
||||||
}
|
|
||||||
|
|
@ -1,199 +0,0 @@
|
||||||
#include "llama.h"
|
|
||||||
#include "common.h"
|
|
||||||
#include "console.h"
|
|
||||||
|
|
||||||
#include <cstdio>
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
|
||||||
#include <vector>
|
|
||||||
#include <fstream>
|
|
||||||
|
|
||||||
// generate using test-tokenizer-0-falcon.py
|
|
||||||
static const std::map<std::string, std::vector<llama_token>> & k_tests() {
|
|
||||||
static std::map<std::string, std::vector<llama_token>> _k_tests = {
|
|
||||||
{ "" , { }, },
|
|
||||||
{ " " , { 204, }, },
|
|
||||||
{ " " , { 258, }, },
|
|
||||||
{ " " , { 466, }, },
|
|
||||||
{ "\t" , { 192, }, },
|
|
||||||
{ "\n" , { 193, }, },
|
|
||||||
{ "\n\n" , { 1001, }, },
|
|
||||||
{ "\n\n\n" , { 11331, }, },
|
|
||||||
{ "\t\n" , { 19125, }, },
|
|
||||||
{ "Hello world" , { 9856, 1079, }, },
|
|
||||||
{ " Hello world" , { 23090, 1079, }, },
|
|
||||||
{ "Hello World" , { 9856, 2889, }, },
|
|
||||||
{ " Hello World" , { 23090, 2889, }, },
|
|
||||||
{ " Hello World!" , { 23090, 2889, 12, }, },
|
|
||||||
{ "Hello, world!" , { 9856, 23, 1079, 12, }, },
|
|
||||||
{ " Hello, world!" , { 23090, 23, 1079, 12, }, },
|
|
||||||
{ " this is 🦙.cpp" , { 414, 304, 3346, 111, 231, 25, 29247, }, },
|
|
||||||
{ "w048 7tuijk dsdfhu" , { 98, 55866, 204, 34, 16682, 7149, 36190, 6869, 11481, }, },
|
|
||||||
{ "нещо на Български" , { 150, 133, 6207, 151, 215, 150, 134, 5052, 133, 6279, 5052, 223, 151, 216, 49679, 123, 53110, 47043, 7795, }, },
|
|
||||||
{ "កាន់តែពិសេសអាចខលចេញ" , { 38154, 206, 38154, 126, 38154, 225, 167, 237, 217, 38154, 221, 167, 237, 208, 38154, 228, 38154, 127, 38154, 237, 167, 237, 207, 38154, 237, 38154, 107, 38154, 126, 38154, 211, 38154, 207, 38154, 233, 38154, 211, 167, 237, 207, 38154, 215, }, },
|
|
||||||
{ "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)", { 2571, 232, 206, 204, 19, 11003, 20, 8196, 126, 283, 219, 48778, 116, 13392, 204, 19, 51831, 732, 63209, 1741, 7955, 522, 20, 22438, 211, 204, 19, 7927, 53360, 325, 504, 701, 946, 10930, 20, }, },
|
|
||||||
{ "Hello" , { 9856, }, },
|
|
||||||
{ " Hello" , { 23090, }, },
|
|
||||||
{ " Hello" , { 204, 23090, }, },
|
|
||||||
{ " Hello" , { 258, 23090, }, },
|
|
||||||
{ " Hello" , { 466, 23090, }, },
|
|
||||||
{ " Hello\n Hello" , { 466, 23090, 742, 23090, }, },
|
|
||||||
{ " (" , { 204, 19, }, },
|
|
||||||
{ "\n =" , { 1212, 40, }, },
|
|
||||||
{ "' era" , { 18, 4932, }, },
|
|
||||||
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 9856, 23, 291, 18, 436, 12, 1265, 362, 299, 8196, 207, 204, 42, 50087, 123, 2727, 20300, 32022, 133, 234, 17419, 30137, 28, 7858, 181, 133, 236, }, },
|
|
||||||
{ "3" , { 30, }, },
|
|
||||||
{ "33" , { 3138, }, },
|
|
||||||
{ "333" , { 22287, }, },
|
|
||||||
{ "3333" , { 22287, 30, }, },
|
|
||||||
{ "33333" , { 22287, 3138, }, },
|
|
||||||
{ "333333" , { 22287, 22287, }, },
|
|
||||||
{ "3333333" , { 22287, 22287, 30, }, },
|
|
||||||
{ "33333333" , { 22287, 22287, 3138, }, },
|
|
||||||
{ "333333333" , { 22287, 22287, 22287, }, },
|
|
||||||
};
|
|
||||||
|
|
||||||
return _k_tests;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
|
||||||
if (argc < 2) {
|
|
||||||
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string fname = argv[1];
|
|
||||||
|
|
||||||
std::string fname_text;
|
|
||||||
if (argc > 2) {
|
|
||||||
fname_text = argv[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
|
||||||
|
|
||||||
llama_model * model;
|
|
||||||
llama_context * ctx;
|
|
||||||
|
|
||||||
llama_backend_init();
|
|
||||||
|
|
||||||
// load the vocab
|
|
||||||
{
|
|
||||||
auto mparams = llama_model_default_params();
|
|
||||||
|
|
||||||
mparams.vocab_only = true;
|
|
||||||
|
|
||||||
model = llama_load_model_from_file(fname.c_str(), mparams);
|
|
||||||
|
|
||||||
if (model == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto cparams = llama_context_default_params();
|
|
||||||
|
|
||||||
ctx = llama_new_context_with_model(model, cparams);
|
|
||||||
|
|
||||||
if (ctx == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
llama_free_model(model);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) {
|
|
||||||
fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__);
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
return 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef _WIN32
|
|
||||||
// We need this for unicode console support
|
|
||||||
console::init(false, false);
|
|
||||||
atexit([]() { console::cleanup(); });
|
|
||||||
#endif
|
|
||||||
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
for (const auto & test_kv : k_tests()) {
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, false);
|
|
||||||
|
|
||||||
printf("\n");
|
|
||||||
printf("src: '%s'\n", test_kv.first.c_str());
|
|
||||||
printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str());
|
|
||||||
printf("tok: ");
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
printf("%d ", tok);
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
|
|
||||||
bool correct = res.size() == test_kv.second.size();
|
|
||||||
for (int i = 0; i < (int) res.size() && correct; ++i) {
|
|
||||||
if (test_kv.second[i] != res[i]) {
|
|
||||||
correct = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!correct) {
|
|
||||||
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
|
|
||||||
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
|
|
||||||
llama_detokenize_bpe(ctx, res).c_str(),
|
|
||||||
llama_detokenize_bpe(ctx, test_kv.second).c_str());
|
|
||||||
fprintf(stderr, "%s : expected tokens: ", __func__);
|
|
||||||
for (const auto & t : test_kv.second) {
|
|
||||||
fprintf(stderr, "%6d, ", t);
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
fprintf(stderr, "%s : got tokens: ", __func__);
|
|
||||||
for (const auto & t : res) {
|
|
||||||
fprintf(stderr, "%6d, ", t);
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
|
|
||||||
success = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!fname_text.empty()) {
|
|
||||||
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
|
|
||||||
|
|
||||||
std::string text;
|
|
||||||
{
|
|
||||||
std::ifstream ifs(fname_text);
|
|
||||||
if (!ifs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
|
|
||||||
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, text, false);
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
|
|
||||||
|
|
||||||
{
|
|
||||||
const std::string fname_out = fname_text + ".tokcpp";
|
|
||||||
|
|
||||||
std::ofstream ofs(fname_out);
|
|
||||||
if (!ofs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
ofs << tok << " '" << llama_detokenize_bpe(ctx, std::vector<int>{tok}) << "'" << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
|
|
||||||
}
|
|
||||||
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
|
|
||||||
llama_backend_free();
|
|
||||||
|
|
||||||
return success ? 0 : 3;
|
|
||||||
}
|
|
||||||
|
|
@ -1,199 +0,0 @@
|
||||||
#include "llama.h"
|
|
||||||
#include "common.h"
|
|
||||||
#include "console.h"
|
|
||||||
|
|
||||||
#include <cstdio>
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
|
||||||
#include <vector>
|
|
||||||
#include <fstream>
|
|
||||||
|
|
||||||
// generate using test-tokenizer-0-llama.py
|
|
||||||
static const std::map<std::string, std::vector<llama_token>> & k_tests() {
|
|
||||||
static std::map<std::string, std::vector<llama_token>> _k_tests = {
|
|
||||||
{ "" , { }, },
|
|
||||||
{ " " , { 220, }, },
|
|
||||||
{ " " , { 256, }, },
|
|
||||||
{ " " , { 262, }, },
|
|
||||||
{ "\t" , { 197, }, },
|
|
||||||
{ "\n" , { 198, }, },
|
|
||||||
{ "\n\n" , { 271, }, },
|
|
||||||
{ "\n\n\n" , { 1432, }, },
|
|
||||||
{ "\t\n" , { 1602, }, },
|
|
||||||
{ "Hello world" , { 9906, 1917, }, },
|
|
||||||
{ " Hello world" , { 22691, 1917, }, },
|
|
||||||
{ "Hello World" , { 9906, 4435, }, },
|
|
||||||
{ " Hello World" , { 22691, 4435, }, },
|
|
||||||
{ " Hello World!" , { 22691, 4435, 0, }, },
|
|
||||||
{ "Hello, world!" , { 9906, 11, 1917, 0, }, },
|
|
||||||
{ " Hello, world!" , { 22691, 11, 1917, 0, }, },
|
|
||||||
{ " this is 🦙.cpp" , { 420, 374, 11410, 99, 247, 13, 11055, }, },
|
|
||||||
{ "w048 7tuijk dsdfhu" , { 86, 23904, 220, 22, 83, 2005, 42908, 11729, 3013, 17156, }, },
|
|
||||||
{ "нещо на Български" , { 79862, 102118, 13373, 64571, 34694, 3114, 112203, 80112, }, },
|
|
||||||
{ "កាន់តែពិសេសអាចខលចេញ" , { 21549, 222, 98629, 241, 45358, 233, 21549, 237, 45358, 224, 21549, 244, 21549, 115, 21549, 253, 45358, 223, 21549, 253, 21549, 95, 98629, 227, 21549, 223, 21549, 249, 21549, 227, 45358, 223, 21549, 231, }, },
|
|
||||||
{ "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)", { 9468, 248, 222, 320, 8416, 8, 27623, 114, 102470, 9468, 234, 104, 31643, 320, 36773, 100166, 98634, 8, 26602, 227, 320, 3323, 43465, 430, 706, 1202, 1866, 4037, 8, }, },
|
|
||||||
{ "Hello" , { 9906, }, },
|
|
||||||
{ " Hello" , { 22691, }, },
|
|
||||||
{ " Hello" , { 220, 22691, }, },
|
|
||||||
{ " Hello" , { 256, 22691, }, },
|
|
||||||
{ " Hello" , { 262, 22691, }, },
|
|
||||||
{ " Hello\n Hello" , { 262, 22691, 198, 262, 22691, }, },
|
|
||||||
{ " (" , { 320, }, },
|
|
||||||
{ "\n =" , { 198, 284, }, },
|
|
||||||
{ "' era" , { 6, 11639, }, },
|
|
||||||
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 9906, 11, 379, 65948, 0, 2650, 527, 499, 27623, 223, 949, 37046, 101067, 19000, 23182, 102301, 9263, 18136, 16, 36827, 21909, }, },
|
|
||||||
{ "3" , { 18, }, },
|
|
||||||
{ "33" , { 1644, }, },
|
|
||||||
{ "333" , { 8765, }, },
|
|
||||||
{ "3333" , { 8765, 18, }, },
|
|
||||||
{ "33333" , { 8765, 1644, }, },
|
|
||||||
{ "333333" , { 8765, 8765, }, },
|
|
||||||
{ "3333333" , { 8765, 8765, 18, }, },
|
|
||||||
{ "33333333" , { 8765, 8765, 1644, }, },
|
|
||||||
{ "333333333" , { 8765, 8765, 8765, }, },
|
|
||||||
};
|
|
||||||
|
|
||||||
return _k_tests;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
|
||||||
if (argc < 2) {
|
|
||||||
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string fname = argv[1];
|
|
||||||
|
|
||||||
std::string fname_text;
|
|
||||||
if (argc > 2) {
|
|
||||||
fname_text = argv[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
|
||||||
|
|
||||||
llama_model * model;
|
|
||||||
llama_context * ctx;
|
|
||||||
|
|
||||||
llama_backend_init();
|
|
||||||
|
|
||||||
// load the vocab
|
|
||||||
{
|
|
||||||
auto mparams = llama_model_default_params();
|
|
||||||
|
|
||||||
mparams.vocab_only = true;
|
|
||||||
|
|
||||||
model = llama_load_model_from_file(fname.c_str(), mparams);
|
|
||||||
|
|
||||||
if (model == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto cparams = llama_context_default_params();
|
|
||||||
|
|
||||||
ctx = llama_new_context_with_model(model, cparams);
|
|
||||||
|
|
||||||
if (ctx == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
llama_free_model(model);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_BPE) {
|
|
||||||
fprintf(stderr, "%s : error: vocab type is not BPE\n", __func__);
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
return 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef _WIN32
|
|
||||||
// We need this for unicode console support
|
|
||||||
console::init(false, false);
|
|
||||||
atexit([]() { console::cleanup(); });
|
|
||||||
#endif
|
|
||||||
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
for (const auto & test_kv : k_tests()) {
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, false);
|
|
||||||
|
|
||||||
printf("\n");
|
|
||||||
printf("src: '%s'\n", test_kv.first.c_str());
|
|
||||||
printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str());
|
|
||||||
printf("tok: ");
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
printf("%d ", tok);
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
|
|
||||||
bool correct = res.size() == test_kv.second.size();
|
|
||||||
for (int i = 0; i < (int) res.size() && correct; ++i) {
|
|
||||||
if (test_kv.second[i] != res[i]) {
|
|
||||||
correct = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!correct) {
|
|
||||||
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
|
|
||||||
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
|
|
||||||
llama_detokenize_bpe(ctx, res).c_str(),
|
|
||||||
llama_detokenize_bpe(ctx, test_kv.second).c_str());
|
|
||||||
fprintf(stderr, "%s : expected tokens: ", __func__);
|
|
||||||
for (const auto & t : test_kv.second) {
|
|
||||||
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
fprintf(stderr, "%s : got tokens: ", __func__);
|
|
||||||
for (const auto & t : res) {
|
|
||||||
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
|
|
||||||
success = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!fname_text.empty()) {
|
|
||||||
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
|
|
||||||
|
|
||||||
std::string text;
|
|
||||||
{
|
|
||||||
std::ifstream ifs(fname_text);
|
|
||||||
if (!ifs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
|
|
||||||
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, text, false);
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
|
|
||||||
|
|
||||||
{
|
|
||||||
const std::string fname_out = fname_text + ".tokcpp";
|
|
||||||
|
|
||||||
std::ofstream ofs(fname_out);
|
|
||||||
if (!ofs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
ofs << tok << " '" << string_strip(llama_detokenize_bpe(ctx, std::vector<int>{tok})) << "'" << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
|
|
||||||
}
|
|
||||||
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
|
|
||||||
llama_backend_free();
|
|
||||||
|
|
||||||
return success ? 0 : 3;
|
|
||||||
}
|
|
||||||
|
|
@ -1,204 +0,0 @@
|
||||||
#include "llama.h"
|
|
||||||
#include "common.h"
|
|
||||||
#include "console.h"
|
|
||||||
|
|
||||||
#include <cstdio>
|
|
||||||
#include <string>
|
|
||||||
#include <map>
|
|
||||||
#include <vector>
|
|
||||||
#include <fstream>
|
|
||||||
|
|
||||||
// generate using test-tokenizer-0-llama.py
|
|
||||||
static const std::map<std::string, std::vector<llama_token>> & k_tests() {
|
|
||||||
static std::map<std::string, std::vector<llama_token>> _k_tests = {
|
|
||||||
{ "" , { }, },
|
|
||||||
{ " " , { 259, }, },
|
|
||||||
{ " " , { 1678, }, },
|
|
||||||
{ " " , { 268, }, },
|
|
||||||
{ "\t" , { 29871, 12, }, },
|
|
||||||
{ "\n" , { 29871, 13, }, },
|
|
||||||
{ "\n\n" , { 29871, 13, 13, }, },
|
|
||||||
{ "\n\n\n" , { 29871, 13, 13, 13, }, },
|
|
||||||
{ "\t\n" , { 29871, 12, 13, }, },
|
|
||||||
{ "Hello world" , { 15043, 3186, }, },
|
|
||||||
{ " Hello world" , { 29871, 15043, 3186, }, },
|
|
||||||
{ "Hello World" , { 15043, 2787, }, },
|
|
||||||
{ " Hello World" , { 29871, 15043, 2787, }, },
|
|
||||||
{ " Hello World!" , { 29871, 15043, 2787, 29991, }, },
|
|
||||||
{ "Hello, world!" , { 15043, 29892, 3186, 29991, }, },
|
|
||||||
{ " Hello, world!" , { 29871, 15043, 29892, 3186, 29991, }, },
|
|
||||||
{ " this is 🦙.cpp" , { 29871, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, },
|
|
||||||
{ "w048 7tuijk dsdfhu" , { 281, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, },
|
|
||||||
{ "нещо на Български" , { 1538, 4851, 665, 1386, 29713, 1305, }, },
|
|
||||||
{ "កាន់តែពិសេសអាចខលចេញ" , { 29871, 31849, 31324, 31934, 228, 162, 142, 228, 161, 146, 228, 162, 133, 228, 161, 153, 228, 161, 186, 31708, 228, 162, 132, 31708, 228, 161, 165, 31324, 228, 161, 136, 228, 161, 132, 228, 161, 158, 228, 161, 136, 228, 162, 132, 228, 161, 140, }, },
|
|
||||||
{ "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)", { 29871, 243, 162, 157, 131, 313, 8945, 29897, 29871, 243, 162, 155, 185, 30722, 243, 162, 143, 174, 30598, 313, 20787, 953, 3848, 275, 16125, 630, 29897, 29871, 31681, 313, 6194, 953, 29877, 2397, 393, 756, 967, 1914, 5993, 29897, }, },
|
|
||||||
{ "Hello" , { 15043, }, },
|
|
||||||
{ " Hello" , { 29871, 15043, }, },
|
|
||||||
{ " Hello" , { 259, 15043, }, },
|
|
||||||
{ " Hello" , { 1678, 15043, }, },
|
|
||||||
{ " Hello" , { 268, 15043, }, },
|
|
||||||
{ " Hello\n Hello" , { 268, 15043, 13, 1678, 15043, }, },
|
|
||||||
{ " (" , { 29871, 313, }, },
|
|
||||||
{ "\n =" , { 29871, 13, 353, }, },
|
|
||||||
{ "' era" , { 525, 3152, }, },
|
|
||||||
{ "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 15043, 29892, 343, 29915, 497, 29991, 1128, 526, 366, 29871, 243, 162, 155, 132, 1577, 30672, 31522, 30505, 11548, 31041, 30732, 29896, 29941, 29896, 29946, 29896, 29945, 29896, 30408, 30739, }, },
|
|
||||||
{ "3" , { 29871, 29941, }, },
|
|
||||||
{ "33" , { 29871, 29941, 29941, }, },
|
|
||||||
{ "333" , { 29871, 29941, 29941, 29941, }, },
|
|
||||||
{ "3333" , { 29871, 29941, 29941, 29941, 29941, }, },
|
|
||||||
{ "33333" , { 29871, 29941, 29941, 29941, 29941, 29941, }, },
|
|
||||||
{ "333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, }, },
|
|
||||||
{ "3333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, },
|
|
||||||
{ "33333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, },
|
|
||||||
{ "333333333" , { 29871, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, 29941, }, },
|
|
||||||
};
|
|
||||||
|
|
||||||
return _k_tests;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
|
||||||
if (argc < 2) {
|
|
||||||
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::string fname = argv[1];
|
|
||||||
|
|
||||||
std::string fname_text;
|
|
||||||
if (argc > 2) {
|
|
||||||
fname_text = argv[2];
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
|
||||||
|
|
||||||
llama_model * model;
|
|
||||||
llama_context * ctx;
|
|
||||||
|
|
||||||
llama_backend_init();
|
|
||||||
|
|
||||||
// load the vocab
|
|
||||||
{
|
|
||||||
auto mparams = llama_model_default_params();
|
|
||||||
|
|
||||||
mparams.vocab_only = true;
|
|
||||||
|
|
||||||
model = llama_load_model_from_file(fname.c_str(), mparams);
|
|
||||||
|
|
||||||
if (model == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto cparams = llama_context_default_params();
|
|
||||||
|
|
||||||
ctx = llama_new_context_with_model(model, cparams);
|
|
||||||
|
|
||||||
if (ctx == NULL) {
|
|
||||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
|
||||||
llama_free_model(model);
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (llama_vocab_type(model) != LLAMA_VOCAB_TYPE_SPM) {
|
|
||||||
fprintf(stderr, "%s : error: vocab type is not SPM\n", __func__);
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
return 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef _WIN32
|
|
||||||
// We need this for unicode console support
|
|
||||||
console::init(false, false);
|
|
||||||
atexit([]() { console::cleanup(); });
|
|
||||||
#endif
|
|
||||||
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
for (const auto & test_kv : k_tests()) {
|
|
||||||
const std::vector<llama_token> res_bos = llama_tokenize(ctx, test_kv.first, true);
|
|
||||||
const std::vector<llama_token> res_nobos = llama_tokenize(ctx, test_kv.first, false);
|
|
||||||
|
|
||||||
printf("\n");
|
|
||||||
printf("src: '%s'\n", test_kv.first.c_str());
|
|
||||||
printf("res: '%s'\n", llama_detokenize_spm(ctx, res_bos).c_str());
|
|
||||||
printf("tok: ");
|
|
||||||
for (const auto & tok : res_bos) {
|
|
||||||
printf("%d ", tok);
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
|
|
||||||
bool correct = res_nobos.size() == test_kv.second.size() && res_bos.size() == res_nobos.size() + 1 && res_bos[0] == llama_token_bos(model);
|
|
||||||
|
|
||||||
for (int i = 0; i < (int) res_nobos.size() && correct; ++i) {
|
|
||||||
if (test_kv.second[i] != res_bos[i + 1]) {
|
|
||||||
correct = false;
|
|
||||||
}
|
|
||||||
if (test_kv.second[i] != res_nobos[i]) {
|
|
||||||
correct = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!correct) {
|
|
||||||
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
|
|
||||||
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
|
|
||||||
llama_detokenize_spm(ctx, res_nobos).c_str(),
|
|
||||||
llama_detokenize_spm(ctx, test_kv.second).c_str());
|
|
||||||
fprintf(stderr, "%s : expected tokens: ", __func__);
|
|
||||||
for (const auto & t : test_kv.second) {
|
|
||||||
fprintf(stderr, "%6d, ", t);
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
fprintf(stderr, "%s : got tokens: ", __func__);
|
|
||||||
for (const auto & t : res_nobos) {
|
|
||||||
fprintf(stderr, "%6d, ", t);
|
|
||||||
}
|
|
||||||
fprintf(stderr, "\n");
|
|
||||||
|
|
||||||
success = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!fname_text.empty()) {
|
|
||||||
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
|
|
||||||
|
|
||||||
std::string text;
|
|
||||||
{
|
|
||||||
std::ifstream ifs(fname_text);
|
|
||||||
if (!ifs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
|
|
||||||
|
|
||||||
const std::vector<llama_token> res = llama_tokenize(ctx, text, true);
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
|
|
||||||
|
|
||||||
{
|
|
||||||
const std::string fname_out = fname_text + ".tokcpp";
|
|
||||||
|
|
||||||
std::ofstream ofs(fname_out);
|
|
||||||
if (!ofs) {
|
|
||||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const auto & tok : res) {
|
|
||||||
ofs << tok << " '" << llama_detokenize_spm(ctx, std::vector<int>{tok}) << "'" << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
|
|
||||||
}
|
|
||||||
|
|
||||||
llama_free_model(model);
|
|
||||||
llama_free(ctx);
|
|
||||||
|
|
||||||
llama_backend_free();
|
|
||||||
|
|
||||||
return success ? 0 : 3;
|
|
||||||
}
|
|
||||||
263
tests/test-tokenizer-0.cpp
Normal file
263
tests/test-tokenizer-0.cpp
Normal file
|
|
@ -0,0 +1,263 @@
|
||||||
|
#include "llama.h"
|
||||||
|
#include "common.h"
|
||||||
|
#include "console.h"
|
||||||
|
|
||||||
|
#include <cstdio>
|
||||||
|
#include <string>
|
||||||
|
#include <map>
|
||||||
|
#include <vector>
|
||||||
|
#include <fstream>
|
||||||
|
|
||||||
|
//static const std::map<std::string, std::vector<llama_token>> & k_tests() {
|
||||||
|
// static std::map<std::string, std::vector<llama_token>> _k_tests = {
|
||||||
|
// { "" , { }, },
|
||||||
|
// { " " , { 220, }, },
|
||||||
|
// { " " , { 256, }, },
|
||||||
|
// { " " , { 262, }, },
|
||||||
|
// { "\t" , { 197, }, },
|
||||||
|
// { "\n" , { 198, }, },
|
||||||
|
// { "\n\n" , { 271, }, },
|
||||||
|
// { "\n\n\n" , { 1432, }, },
|
||||||
|
// { "\t\n" , { 1602, }, },
|
||||||
|
// { "Hello world" , { 9906, 1917, }, },
|
||||||
|
// { " Hello world" , { 22691, 1917, }, },
|
||||||
|
// { "Hello World" , { 9906, 4435, }, },
|
||||||
|
// { " Hello World" , { 22691, 4435, }, },
|
||||||
|
// { " Hello World!" , { 22691, 4435, 0, }, },
|
||||||
|
// { "Hello, world!" , { 9906, 11, 1917, 0, }, },
|
||||||
|
// { " Hello, world!" , { 22691, 11, 1917, 0, }, },
|
||||||
|
// { " this is 🦙.cpp" , { 420, 374, 11410, 99, 247, 13, 11055, }, },
|
||||||
|
// { "w048 7tuijk dsdfhu" , { 86, 23904, 220, 22, 83, 2005, 42908, 11729, 3013, 17156, }, },
|
||||||
|
// { "нещо на Български" , { 79862, 102118, 13373, 64571, 34694, 3114, 112203, 80112, }, },
|
||||||
|
// { "កាន់តែពិសេសអាចខលចេញ" , { 21549, 222, 98629, 241, 45358, 233, 21549, 237, 45358, 224, 21549, 244, 21549, 115, 21549, 253, 45358, 223, 21549, 253, 21549, 95, 98629, 227, 21549, 223, 21549, 249, 21549, 227, 45358, 223, 21549, 231, }, },
|
||||||
|
// { "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)", { 9468, 248, 222, 320, 8416, 8, 27623, 114, 102470, 9468, 234, 104, 31643, 320, 36773, 100166, 98634, 8, 26602, 227, 320, 3323, 43465, 430, 706, 1202, 1866, 4037, 8, }, },
|
||||||
|
// { "Hello" , { 9906, }, },
|
||||||
|
// { " Hello" , { 22691, }, },
|
||||||
|
// { " Hello" , { 220, 22691, }, },
|
||||||
|
// { " Hello" , { 256, 22691, }, },
|
||||||
|
// { " Hello" , { 262, 22691, }, },
|
||||||
|
// { " Hello\n Hello" , { 262, 22691, 198, 262, 22691, }, },
|
||||||
|
// { " (" , { 320, }, },
|
||||||
|
// { "\n =" , { 198, 284, }, },
|
||||||
|
// { "' era" , { 6, 11639, }, },
|
||||||
|
// { "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 9906, 11, 379, 65948, 0, 2650, 527, 499, 27623, 223, 949, 37046, 101067, 19000, 23182, 102301, 9263, 18136, 16, 36827, 21909, }, },
|
||||||
|
// { "3" , { 18, }, },
|
||||||
|
// { "33" , { 1644, }, },
|
||||||
|
// { "333" , { 8765, }, },
|
||||||
|
// { "3333" , { 8765, 18, }, },
|
||||||
|
// { "33333" , { 8765, 1644, }, },
|
||||||
|
// { "333333" , { 8765, 8765, }, },
|
||||||
|
// { "3333333" , { 8765, 8765, 18, }, },
|
||||||
|
// { "33333333" , { 8765, 8765, 1644, }, },
|
||||||
|
// { "333333333" , { 8765, 8765, 8765, }, },
|
||||||
|
// };
|
||||||
|
//
|
||||||
|
// return _k_tests;
|
||||||
|
//}
|
||||||
|
|
||||||
|
static std::map<std::string, std::vector<llama_token>> read_tests(const std::string & fname_inp, const std::string & fname_out) {
|
||||||
|
std::map<std::string, std::vector<llama_token>> tests;
|
||||||
|
|
||||||
|
std::ifstream ifs_inp(fname_inp);
|
||||||
|
if (!ifs_inp) {
|
||||||
|
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_inp.c_str());
|
||||||
|
return tests;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string sraw((std::istreambuf_iterator<char>(ifs_inp)), std::istreambuf_iterator<char>());
|
||||||
|
|
||||||
|
std::ifstream ifs_out(fname_out);
|
||||||
|
if (!ifs_out) {
|
||||||
|
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
||||||
|
return tests;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::string> sout;
|
||||||
|
for (std::string line; std::getline(ifs_out, line);) {
|
||||||
|
sout.push_back(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string sep = "\n__ggml_vocab_test__\n";
|
||||||
|
|
||||||
|
std::vector<std::string> sinp;
|
||||||
|
|
||||||
|
size_t pos = 0;
|
||||||
|
while (pos < sraw.size()) {
|
||||||
|
const size_t next = sraw.find(sep, pos);
|
||||||
|
if (next == std::string::npos) {
|
||||||
|
sinp.push_back(sraw.substr(pos));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
sinp.push_back(sraw.substr(pos, next - pos));
|
||||||
|
pos = next + sep.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sinp.size() != sout.size()) {
|
||||||
|
fprintf(stderr, "%s : error: input and output files have different number of tests\n", __func__);
|
||||||
|
return tests;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (size_t i = 0; i < sinp.size(); ++i) {
|
||||||
|
const std::string & s = sinp[i];
|
||||||
|
const std::string & o = string_strip(sout[i]);
|
||||||
|
|
||||||
|
std::vector<llama_token> toks;
|
||||||
|
|
||||||
|
size_t pos = 0;
|
||||||
|
while (pos < o.size()) {
|
||||||
|
size_t next = o.find(' ', pos);
|
||||||
|
if (next == std::string::npos) {
|
||||||
|
next = o.size();
|
||||||
|
}
|
||||||
|
const std::string stok = o.substr(pos, next - pos);
|
||||||
|
toks.push_back(std::stoi(stok));
|
||||||
|
pos = next + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
tests[s] = toks;
|
||||||
|
}
|
||||||
|
|
||||||
|
return tests;
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv) {
|
||||||
|
if (argc < 2) {
|
||||||
|
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string fname = argv[1];
|
||||||
|
|
||||||
|
const std::string fname_inp = fname + ".inp";
|
||||||
|
const std::string fname_out = fname + ".out";
|
||||||
|
|
||||||
|
std::string fname_text;
|
||||||
|
if (argc > 2) {
|
||||||
|
fname_text = argv[2];
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
||||||
|
|
||||||
|
llama_model * model;
|
||||||
|
llama_context * ctx;
|
||||||
|
|
||||||
|
llama_backend_init();
|
||||||
|
|
||||||
|
// load the vocab
|
||||||
|
{
|
||||||
|
auto mparams = llama_model_default_params();
|
||||||
|
|
||||||
|
mparams.vocab_only = true;
|
||||||
|
|
||||||
|
model = llama_load_model_from_file(fname.c_str(), mparams);
|
||||||
|
|
||||||
|
if (model == NULL) {
|
||||||
|
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto cparams = llama_context_default_params();
|
||||||
|
|
||||||
|
ctx = llama_new_context_with_model(model, cparams);
|
||||||
|
|
||||||
|
if (ctx == NULL) {
|
||||||
|
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||||
|
llama_free_model(model);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef _WIN32
|
||||||
|
// We need this for unicode console support
|
||||||
|
console::init(false, false);
|
||||||
|
atexit([]() { console::cleanup(); });
|
||||||
|
#endif
|
||||||
|
|
||||||
|
bool success = true;
|
||||||
|
|
||||||
|
const auto k_tests = read_tests(fname_inp, fname_out);
|
||||||
|
|
||||||
|
const bool add_special = false;
|
||||||
|
|
||||||
|
for (const auto & test_kv : k_tests) {
|
||||||
|
const std::vector<llama_token> res = llama_tokenize(ctx, test_kv.first, add_special);
|
||||||
|
|
||||||
|
printf("\n");
|
||||||
|
printf("src: '%s'\n", test_kv.first.c_str());
|
||||||
|
printf("res: '%s'\n", llama_detokenize_bpe(ctx, res).c_str());
|
||||||
|
printf("tok: ");
|
||||||
|
for (const auto & tok : res) {
|
||||||
|
printf("%d ", tok);
|
||||||
|
}
|
||||||
|
printf("\n");
|
||||||
|
|
||||||
|
bool correct = res.size() == test_kv.second.size();
|
||||||
|
for (int i = 0; i < (int) res.size() && correct; ++i) {
|
||||||
|
if (test_kv.second[i] != res[i]) {
|
||||||
|
correct = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!correct) {
|
||||||
|
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
|
||||||
|
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
|
||||||
|
llama_detokenize_bpe(ctx, res).c_str(),
|
||||||
|
llama_detokenize_bpe(ctx, test_kv.second).c_str());
|
||||||
|
fprintf(stderr, "%s : expected tokens: ", __func__);
|
||||||
|
for (const auto & t : test_kv.second) {
|
||||||
|
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
|
||||||
|
}
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
fprintf(stderr, "%s : got tokens: ", __func__);
|
||||||
|
for (const auto & t : res) {
|
||||||
|
fprintf(stderr, "%6d '%s', ", t, llama_token_to_piece(ctx, t).c_str());
|
||||||
|
}
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
|
||||||
|
success = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fname_text.empty()) {
|
||||||
|
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
|
||||||
|
|
||||||
|
std::string text;
|
||||||
|
{
|
||||||
|
std::ifstream ifs(fname_text);
|
||||||
|
if (!ifs) {
|
||||||
|
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
|
||||||
|
|
||||||
|
const std::vector<llama_token> res = llama_tokenize(ctx, text, add_special);
|
||||||
|
|
||||||
|
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
|
||||||
|
|
||||||
|
{
|
||||||
|
const std::string fname_out = fname_text + ".tokcpp";
|
||||||
|
|
||||||
|
std::ofstream ofs(fname_out);
|
||||||
|
if (!ofs) {
|
||||||
|
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const auto & tok : res) {
|
||||||
|
ofs << tok << " '" << string_strip(llama_detokenize_bpe(ctx, std::vector<int>{tok})) << "'" << std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
|
||||||
|
}
|
||||||
|
|
||||||
|
llama_free_model(model);
|
||||||
|
llama_free(ctx);
|
||||||
|
|
||||||
|
llama_backend_free();
|
||||||
|
|
||||||
|
return success ? 0 : 3;
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue