vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cont [no ci]

Browse source
This commit is contained in:
Georgi Gerganov 2024-12-15 12:36:03 +02:00
parent 7a20c287c7
commit 3e92f4ecbe
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/unicode.h
View file

@ -49,13 +49,13 @@ struct unicode_cpt_flags {
size_t unicode_len_utf8(char src); size_t unicode_len_utf8(char src);
std::string unicode_cpt_to_utf8 (uint32_t cpt); std::string unicode_cpt_to_utf8 (uint32_t cpt);
uint32_t unicode_cpt_from_utf8 (const std::string & utf8, size_t & offset); uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8); std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts); std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);
unicode_cpt_flags unicode_cpt_flags_from_cpt(uint32_t cpt); unicode_cpt_flags unicode_cpt_flags_from_cpt (uint32_t cpt);
unicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8); unicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8);
std::string unicode_byte_to_utf8(uint8_t byte); std::string unicode_byte_to_utf8(uint8_t byte);