vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Simplify to_lower cases

Browse source 
Co-authored-by: Jared Van Bortel <cebtenzzre@gmail.com>
This commit is contained in:
Douglas Hanley 2024-02-27 14:08:39 -05:00 • committed by GitHub
parent 9242cf1421
commit 801abe52a3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama.cpp
View file

@ -8961,10 +8961,8 @@ struct llm_tokenizer_wpm {
if (code > 0xFFFF) { if (code > 0xFFFF) {
return code; return code;
} }
return std::tolower(static_cast<wchar_t>(code), std::locale("en_US.UTF-8"));
#else
return std::tolower((wchar_t)code, std::locale("en_US.UTF-8"));
#endif #endif
return std::tolower(wchar_t(code), std::locale("en_US.UTF-8"));
} }
bool is_ascii_punct(uint32_t code) { bool is_ascii_punct(uint32_t code) {