vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

unicode : straighten tables

Browse source
This commit is contained in:
Georgi Gerganov 2024-03-11 11:53:17 +02:00
parent de0929ae7d
commit e607540ec9
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
2 changed files with 1386 additions and 507 deletions
Download patch file Download diff file Expand all files Collapse all files

1879
unicode.cpp
View file

File diff suppressed because it is too large Load diff

14
unicode.h
View file

@ -5,13 +5,13 @@
#include <vector> #include <vector>
#define CODEPOINT_TYPE_UNIDENTIFIED 0 #define CODEPOINT_TYPE_UNIDENTIFIED 0
#define CODEPOINT_TYPE_DIGIT 1 #define CODEPOINT_TYPE_DIGIT 1
#define CODEPOINT_TYPE_LETTER 2 #define CODEPOINT_TYPE_LETTER 2
#define CODEPOINT_TYPE_WHITESPACE 3 #define CODEPOINT_TYPE_WHITESPACE 3
#define CODEPOINT_TYPE_ACCENT_MARK 4 #define CODEPOINT_TYPE_ACCENT_MARK 4
#define CODEPOINT_TYPE_PUNCTUATION 5 #define CODEPOINT_TYPE_PUNCTUATION 5
#define CODEPOINT_TYPE_SYMBOL 6 #define CODEPOINT_TYPE_SYMBOL 6
#define CODEPOINT_TYPE_CONTROL 7 #define CODEPOINT_TYPE_CONTROL 7
// TODO: remove // TODO: remove
const std::multimap<uint32_t, uint32_t> & unicode_nfd_map(); const std::multimap<uint32_t, uint32_t> & unicode_nfd_map();