vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

minor : code style normalization

Browse source
This commit is contained in:
Georgi Gerganov 2024-02-11 12:59:59 +02:00
parent 6972e7e90e
commit 8fbefed148
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 17 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

19
llama.cpp
View file

@ -8267,9 +8267,10 @@ struct llm_tokenizer_wpm {
std::string text2 = strip_accents(text);
for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i])) {
char c = text2[i];
if (c >= 'A' && c <= 'Z')
if (c >= 'A' && c <= 'Z') {
text2[i] = c - 'A' + 'a';
}
}
return text2;
}
@ -8312,14 +8313,14 @@ struct llm_tokenizer_wpm {
(codepoint >= 0x2F800 && codepoint <= 0x2FA1F) ||
(codepoint >= 0x3000 && codepoint <= 0x303F) ||
(codepoint >= 0xFF00 && codepoint <= 0xFFEF)) {
return true;
return true; // NOLINT
}
return false;
}
std::string strip_accents(const std::string &inputString) {
std::string strip_accents(const std::string & input_string) {
std::string resultString;
std::map<std::string, char> accentMap = {
std::map<std::string, char> accent_map = {
{"À", 'A'}, {"Á", 'A'}, {"Â", 'A'}, {"Ã", 'A'}, {"Ä", 'A'}, {"Å", 'A'},
{"à", 'a'}, {"á", 'a'}, {"â", 'a'}, {"ã", 'a'}, {"ä", 'a'}, {"å", 'a'},
{"È", 'E'}, {"É", 'E'}, {"Ê", 'E'}, {"Ë", 'E'}, {"è", 'e'}, {"é", 'e'},
@ -8331,11 +8332,11 @@ struct llm_tokenizer_wpm {
{"Ç", 'C'}, {"ç", 'c'}, {"Ñ", 'N'}, {"ñ", 'n'},
};
for (size_t i = 0; i < inputString.length();) {
int len = utf8_len(inputString[i]);
std::string curChar = inputString.substr(i, len);
auto iter = accentMap.find(curChar);
if (iter != accentMap.end()) {
for (size_t i = 0; i < input_string.length();) {
int len = utf8_len(input_string[i]);
std::string curChar = input_string.substr(i, len);
auto iter = accent_map.find(curChar);
if (iter != accent_map.end()) {
resultString += iter->second;
} else {
resultString += curChar;