vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

llama : refactor unicode stuff (#5992)

Browse source 
* llama : refactor unicode stuff

ggml-ci

* unicode : names

* make : fix c++ compiler

* unicode : names

* unicode : straighten tables

* zig : fix build

* unicode : put nfd normalization behind API

ggml-ci

* swift : fix build

* unicode : add BOM

* unicode : add <cstdint>

ggml-ci

* unicode : pass as cpts as const ref
This commit is contained in:
Georgi Gerganov 2024-03-11 17:47:47 +02:00 • committed by GitHub
parent 828defefb6
commit 83796e62bc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 1744 additions and 836 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tests/test-tokenizer-1-bpe.cpp
View file

@ -64,7 +64,7 @@ int main(int argc, char **argv) {
for (int i = 0; i < n_vocab; ++i) {
std::string str = llama_detokenize_bpe(ctx, std::vector<int>(1, i));
try {
auto cps = codepoints_from_utf8(str);
auto cps = unicode_cpts_from_utf8(str);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_bpe(ctx, tokens);
if (check != str) {
@ -97,7 +97,7 @@ int main(int argc, char **argv) {
continue;
}
std::string str = codepoint_to_utf8(cp);
std::string str = unicode_cpt_to_utf8(cp);
std::vector<llama_token> tokens = llama_tokenize(ctx, str, false);
std::string check = llama_detokenize_bpe(ctx, tokens);
if (cp != 9601 && str != check) {