vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update unicode data

Browse source
This commit is contained in:
jaime-m-p 2024-07-20 23:12:08 +02:00
parent ba4bbbd1ad
commit 8f9f05bf6d
2 changed files with 4520 additions and 2275 deletions
Download patch file Download diff file Expand all files Collapse all files

6793
src/unicode-data.cpp
View file

File diff suppressed because it is too large Load diff

2
src/unicode-data.h
View file

@ -13,7 +13,7 @@ struct range_nfd {
static const uint32_t MAX_CODEPOINTS = 0x110000;
extern const std::vector<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;
extern const std::vector<uint16_t> unicode_rle_codepoints_categs;
extern const std::unordered_set<uint32_t> unicode_set_whitespace;
extern const std::unordered_map<uint32_t, uint32_t> unicode_map_lowercase;
extern const std::unordered_map<uint32_t, uint32_t> unicode_map_uppercase;