vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix initialization of static maps

Browse source
This commit is contained in:
goerch 2023-10-01 20:46:06 +02:00 • committed by GitHub
parent 37af613dfc
commit 2117e23f58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 62 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

27
unicode.h
View file

@ -356,9 +356,8 @@ static std::vector<uint32_t> codepoints_from_utf16(const std::vector<uint16_t>&
#define CODEPOINT_TYPE_SYMBOL 6 #define CODEPOINT_TYPE_SYMBOL 6
#define CODEPOINT_TYPE_CONTROL 7 #define CODEPOINT_TYPE_CONTROL 7
static int codepoint_type(uint32_t cp) { static std::unordered_map<uint32_t, int> codepoint_type_map() {
static std::unordered_map<uint32_t, int> codepoint_types; std::unordered_map<uint32_t, int> codepoint_types;
if(codepoint_types.size() == 0) {
for(auto p : digit_ranges) { for(auto p : digit_ranges) {
for(auto i = p.first; i <= p.second; ++ i) for(auto i = p.first; i <= p.second; ++ i)
codepoint_types[i] = CODEPOINT_TYPE_DIGIT; codepoint_types[i] = CODEPOINT_TYPE_DIGIT;
@ -387,7 +386,11 @@ static int codepoint_type(uint32_t cp) {
for(auto i = p.first; i <= p.second; ++ i) for(auto i = p.first; i <= p.second; ++ i)
codepoint_types[i] = CODEPOINT_TYPE_CONTROL; codepoint_types[i] = CODEPOINT_TYPE_CONTROL;
} }
return codepoint_types;
} }
static int codepoint_type(uint32_t cp) {
static std::unordered_map<uint32_t, int> codepoint_types = codepoint_type_map();
return codepoint_types[cp]; return codepoint_types[cp];
} }
@ -398,8 +401,8 @@ static int codepoint_type(std::string utf8) {
return codepoint_type(codepoint_from_utf8(utf8, offset)); return codepoint_type(codepoint_from_utf8(utf8, offset));
} }
static std::string bytes_to_unicode_bpe(uint8_t byte) { static std::unordered_map<uint8_t, std::string> bytes_to_unicode_map_bpe() {
static std::unordered_map<uint8_t, std::string> map; std::unordered_map<uint8_t, std::string> map;
for (int ch = u'!'; ch <= u'~'; ++ch) { for (int ch = u'!'; ch <= u'~'; ++ch) {
map[ch] = codepoint_to_utf8(ch); map[ch] = codepoint_to_utf8(ch);
} }
@ -416,12 +419,16 @@ static std::string bytes_to_unicode_bpe(uint8_t byte) {
++n; ++n;
} }
} }
return map;
}
static std::string bytes_to_unicode_bpe(uint8_t byte) {
static std::unordered_map<uint8_t, std::string> map = bytes_to_unicode_map_bpe();
return map.at(byte); return map.at(byte);
} }
static uint8_t unicode_to_bytes_bpe(const std::string& utf8) { static std::unordered_map<std::string, uint8_t> unicode_to_bytes_map_bpe() {
static std::unordered_map<std::string, uint8_t> map; std::unordered_map<std::string, uint8_t> map;
if (map.size() == 0) {
for (int ch = u'!'; ch <= u'~'; ++ch) { for (int ch = u'!'; ch <= u'~'; ++ch) {
map[codepoint_to_utf8(ch)] = ch; map[codepoint_to_utf8(ch)] = ch;
} }
@ -438,7 +445,11 @@ static uint8_t unicode_to_bytes_bpe(const std::string& utf8) {
++n; ++n;
} }
} }
return map;
} }
static uint8_t unicode_to_bytes_bpe(const std::string& utf8) {
static std::unordered_map<std::string, uint8_t> map = unicode_to_bytes_map_bpe();
return map.at(utf8); return map.at(utf8);
} }