vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Try to fix build problem

Browse source
This commit is contained in:
goerch 2023-09-18 20:15:43 +02:00
parent bfaab6f4fa
commit 89e74c67e2
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
unicode.h
View file

@ -399,13 +399,13 @@ static int codepoint_type(std::string utf8) {
static std::string bytes_to_unicode_bpe(uint8_t byte) { static std::string bytes_to_unicode_bpe(uint8_t byte) {
static std::unordered_map<uint8_t, std::string> map; static std::unordered_map<uint8_t, std::string> map;
for (int ch = u'!'; ch <= u'~'; ++ch) { for (int ch = U'!'; ch <= U'~'; ++ch) {
map[ch] = codepoint_to_utf8(ch); map[ch] = codepoint_to_utf8(ch);
} }
for (int ch = u'¡'; ch <= u'¬'; ++ch) { for (int ch = U'¡'; ch <= U'¬'; ++ch) {
map[ch] = codepoint_to_utf8(ch); map[ch] = codepoint_to_utf8(ch);
} }
for (int ch = u'®'; ch <= u'ÿ'; ++ch) { for (int ch = U'®'; ch <= U'ÿ'; ++ch) {
map[ch] = codepoint_to_utf8(ch); map[ch] = codepoint_to_utf8(ch);
} }
auto n = 0; auto n = 0;
@ -421,13 +421,13 @@ static std::string bytes_to_unicode_bpe(uint8_t byte) {
static uint8_t unicode_to_bytes_bpe(const std::string& utf8) { static uint8_t unicode_to_bytes_bpe(const std::string& utf8) {
static std::unordered_map<std::string, uint8_t> map; static std::unordered_map<std::string, uint8_t> map;
if (map.size() == 0) { if (map.size() == 0) {
for (int ch = u'!'; ch <= u'~'; ++ch) { for (int ch = U'!'; ch <= U'~'; ++ch) {
map[codepoint_to_utf8(ch)] = ch; map[codepoint_to_utf8(ch)] = ch;
} }
for (int ch = u'¡'; ch <= u'¬'; ++ch) { for (int ch = U'¡'; ch <= U'¬'; ++ch) {
map[codepoint_to_utf8(ch)] = ch; map[codepoint_to_utf8(ch)] = ch;
} }
for (int ch = u'®'; ch <= u'ÿ'; ++ch) { for (int ch = U'®'; ch <= U'ÿ'; ++ch) {
map[codepoint_to_utf8(ch)] = ch; map[codepoint_to_utf8(ch)] = ch;
} }
auto n = 0; auto n = 0;