vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'bert' of github.com:iamlemec/llama.cpp into bert

Browse source
This commit is contained in:
Douglas Hanley 2024-02-11 09:51:25 -06:00
commit 61bab4781c
1 changed files with 17 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

19
llama.cpp
View file

@ -8273,9 +8273,10 @@ struct llm_tokenizer_wpm {
std::string text2 = strip_accents(text); std::string text2 = strip_accents(text);
for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i])) { for (size_t i = 0; i < text2.size(); i += utf8_len(text2[i])) {
char c = text2[i]; char c = text2[i];
if (c >= 'A' && c <= 'Z') if (c >= 'A' && c <= 'Z') {
text2[i] = c - 'A' + 'a'; text2[i] = c - 'A' + 'a';
} }
}
return text2; return text2;
} }
@ -8318,14 +8319,14 @@ struct llm_tokenizer_wpm {
(codepoint >= 0x2F800 && codepoint <= 0x2FA1F) || (codepoint >= 0x2F800 && codepoint <= 0x2FA1F) ||
(codepoint >= 0x3000 && codepoint <= 0x303F) || (codepoint >= 0x3000 && codepoint <= 0x303F) ||
(codepoint >= 0xFF00 && codepoint <= 0xFFEF)) { (codepoint >= 0xFF00 && codepoint <= 0xFFEF)) {
return true; return true; // NOLINT
} }
return false; return false;
} }
std::string strip_accents(const std::string &inputString) { std::string strip_accents(const std::string & input_string) {
std::string resultString; std::string resultString;
std::map<std::string, char> accentMap = { std::map<std::string, char> accent_map = {
{"À", 'A'}, {"Á", 'A'}, {"Â", 'A'}, {"Ã", 'A'}, {"Ä", 'A'}, {"Å", 'A'}, {"À", 'A'}, {"Á", 'A'}, {"Â", 'A'}, {"Ã", 'A'}, {"Ä", 'A'}, {"Å", 'A'},
{"à", 'a'}, {"á", 'a'}, {"â", 'a'}, {"ã", 'a'}, {"ä", 'a'}, {"å", 'a'}, {"à", 'a'}, {"á", 'a'}, {"â", 'a'}, {"ã", 'a'}, {"ä", 'a'}, {"å", 'a'},
{"È", 'E'}, {"É", 'E'}, {"Ê", 'E'}, {"Ë", 'E'}, {"è", 'e'}, {"é", 'e'}, {"È", 'E'}, {"É", 'E'}, {"Ê", 'E'}, {"Ë", 'E'}, {"è", 'e'}, {"é", 'e'},
@ -8337,11 +8338,11 @@ struct llm_tokenizer_wpm {
{"Ç", 'C'}, {"ç", 'c'}, {"Ñ", 'N'}, {"ñ", 'n'}, {"Ç", 'C'}, {"ç", 'c'}, {"Ñ", 'N'}, {"ñ", 'n'},
}; };
for (size_t i = 0; i < inputString.length();) { for (size_t i = 0; i < input_string.length();) {
int len = utf8_len(inputString[i]); int len = utf8_len(input_string[i]);
std::string curChar = inputString.substr(i, len); std::string curChar = input_string.substr(i, len);
auto iter = accentMap.find(curChar); auto iter = accent_map.find(curChar);
if (iter != accentMap.end()) { if (iter != accent_map.end()) {
resultString += iter->second; resultString += iter->second;
} else { } else {
resultString += curChar; resultString += curChar;