vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix initialization of static maps

Browse source
This commit is contained in:
goerch 2023-10-01 20:46:06 +02:00 • committed by GitHub
parent 37af613dfc
commit 2117e23f58
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 62 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

31
unicode.h
View file

@ -356,9 +356,8 @@ static std::vector<uint32_t> codepoints_from_utf16(const std::vector<uint16_t>&
#define CODEPOINT_TYPE_SYMBOL 6
#define CODEPOINT_TYPE_CONTROL 7
static int codepoint_type(uint32_t cp) {
static std::unordered_map<uint32_t, int> codepoint_types;
if(codepoint_types.size() == 0) {
static std::unordered_map<uint32_t, int> codepoint_type_map() {
std::unordered_map<uint32_t, int> codepoint_types;
for(auto p : digit_ranges) {
for(auto i = p.first; i <= p.second; ++ i)
codepoint_types[i] = CODEPOINT_TYPE_DIGIT;
@ -387,7 +386,11 @@ static int codepoint_type(uint32_t cp) {
for(auto i = p.first; i <= p.second; ++ i)
codepoint_types[i] = CODEPOINT_TYPE_CONTROL;
}
}
return codepoint_types;
}
static int codepoint_type(uint32_t cp) {
static std::unordered_map<uint32_t, int> codepoint_types = codepoint_type_map();
return codepoint_types[cp];
}
@ -398,8 +401,8 @@ static int codepoint_type(std::string utf8) {
return codepoint_type(codepoint_from_utf8(utf8, offset));
}
static std::string bytes_to_unicode_bpe(uint8_t byte) {
static std::unordered_map<uint8_t, std::string> map;
static std::unordered_map<uint8_t, std::string> bytes_to_unicode_map_bpe() {
std::unordered_map<uint8_t, std::string> map;
for (int ch = u'!'; ch <= u'~'; ++ch) {
map[ch] = codepoint_to_utf8(ch);
}
@ -416,12 +419,16 @@ static std::string bytes_to_unicode_bpe(uint8_t byte) {
++n;
}
}
return map;
}
static std::string bytes_to_unicode_bpe(uint8_t byte) {
static std::unordered_map<uint8_t, std::string> map = bytes_to_unicode_map_bpe();
return map.at(byte);
}
static uint8_t unicode_to_bytes_bpe(const std::string& utf8) {
static std::unordered_map<std::string, uint8_t> map;
if (map.size() == 0) {
static std::unordered_map<std::string, uint8_t> unicode_to_bytes_map_bpe() {
std::unordered_map<std::string, uint8_t> map;
for (int ch = u'!'; ch <= u'~'; ++ch) {
map[codepoint_to_utf8(ch)] = ch;
}
@ -438,7 +445,11 @@ static uint8_t unicode_to_bytes_bpe(const std::string& utf8) {
++n;
}
}
}
return map;
}
static uint8_t unicode_to_bytes_bpe(const std::string& utf8) {
static std::unordered_map<std::string, uint8_t> map = unicode_to_bytes_map_bpe();
return map.at(utf8);
}