vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding unicode regex mappings

Browse source
This commit is contained in:
Kazim Abrar Mahi 2024-04-15 23:48:04 +06:00 • committed by Georgi Gerganov
parent 4c3e882a85
commit a5710a4101
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
2 changed files with 6 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unicode-data.cpp
View file

@ -1649,3 +1649,7 @@ const std::map<char32_t, char32_t> unicode_map_lowercase = {
{0x1E917, 0x1E939}, {0x1E918, 0x1E93A}, {0x1E919, 0x1E93B}, {0x1E91A, 0x1E93C}, {0x1E91B, 0x1E93D}, {0x1E91C, 0x1E93E},
{0x1E91D, 0x1E93F}, {0x1E91E, 0x1E940}, {0x1E91F, 0x1E941}, {0x1E920, 0x1E942}, {0x1E921, 0x1E943},
};
const std::map<std::string, std::wstring> unicode_regex_to_wregex = {
};

2
unicode-data.h
View file

@ -14,3 +14,5 @@ extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;
extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
extern const std::map<char32_t, char32_t> unicode_map_lowercase;
extern const std::map<std::string, std::wstring> unicode_regex_to_wregex;
extern const std::map<std::string, std::string> unicode_regex_to_regex;