vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

unicode : clean-up

Browse source
This commit is contained in:
Georgi Gerganov 2024-04-28 18:01:59 +03:00
parent d63cc9068b
commit e972e6cbf8
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
3 changed files with 53 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

7
unicode-data.h
View file

@ -2,8 +2,6 @@
#include <cstdint>
#include <map>
#include <set>
#include <string>
#include <utility>
#include <vector>
@ -14,6 +12,5 @@ extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_ma
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;
extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
extern const std::map<char32_t, char32_t> unicode_map_lowercase;
extern const std::set<std::string> unicode_regex_with_custom_preprocessor;
extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;
extern const std::map<char32_t, char32_t> unicode_map_lowercase;