vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

dont construct new locale every time

Browse source
This commit is contained in:
Douglas Hanley 2024-02-29 16:08:44 -06:00
parent 4bffc07144
commit 4a24bdfabd
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
llama.cpp
View file

@ -9001,12 +9001,13 @@ struct llm_tokenizer_wpm {
}
uint32_t to_lower(uint32_t code) {
static const std::locale locale("en_US.UTF-8");
#if defined(_WIN32)
if (code > 0xFFFF) {
return code;
}
#endif
return std::tolower(wchar_t(code), std::locale("en_US.UTF-8"));
return std::tolower(wchar_t(code), locale);
}
bool is_ascii_punct(uint32_t code) {