vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: do not insert in the middle of iteration

Browse source
This commit is contained in:
Joan Martinez 2024-05-07 16:56:34 +02:00
parent d6edc627db
commit eb5a0e162a
2 changed files with 21 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

22
unicode.cpp
View file

@ -470,9 +470,7 @@ std::string unicode_cpt_to_utf8(uint32_t cp) {
throw std::invalid_argument("invalid codepoint");
}
// Function to sort subsequences based on canonical class
std::vector<uint32_t> sort_by_canonical_class(std::vector<uint32_t> & cpts) {
auto compareByCanonicalClass = [&](const uint32_t& a, const uint32_t& b) {
auto compareByCanonicalClass = [&](const uint32_t& a, const uint32_t& b) {
auto cc_a_it = unicode_canonical_class.find(a);
if (cc_a_it != unicode_canonical_class.end()) {
auto cc_b_it = unicode_canonical_class.find(b);
@ -482,35 +480,37 @@ std::vector<uint32_t> sort_by_canonical_class(std::vector<uint32_t> & cpts) {
}
return false;
};
};
// Function to sort subsequences based on canonical class
std::vector<uint32_t> sort_by_canonical_class(std::vector<uint32_t> & cpts) {
// Sort the sequence using the custom comparator function
sort(cpts.begin(), cpts.end(), compareByCanonicalClass);
return cpts;
}
std::vector<uint32_t> canonical_decomposition_cpts(std::vector<uint32_t> & cpts, const std::vector<uint32_t>::iterator& cpt_begin, const std::vector<uint32_t>::iterator& cpt_end) {
std::vector<uint32_t> canonical_decomposition_cpts(std::vector<uint32_t> & cpts, uint32_t starting_offset) {
std::vector<uint32_t> result;
for (auto cpt_it = cpt_begin; cpt_it != cpt_end; ++cpt_it) {
auto it = unicode_map_nfd.equal_range(*cpt_it);
for (auto i = starting_offset; i < cpts.size(); i++) {
auto it = unicode_map_nfd.equal_range(cpts[i]);
if (it.first != it.second) {
uint offset = 0;
for (auto jt = it.first; jt != it.second; jt++) {
cpts.insert(cpt_it + offset, jt->second);
cpts.emplace(cpts.begin() + i + offset, jt->second);
offset++;
}
const auto & inner_result = canonical_decomposition_cpts(cpts, cpt_it, cpt_end);
const auto & inner_result = canonical_decomposition_cpts(cpts, i);
result.insert(result.end(), inner_result.begin(), inner_result.end());
break;
} else {
result.push_back(*cpt_it);
result.push_back(cpts[i]);
}
}
return result;
}
std::vector<uint32_t> unicode_cpts_normalize_nfd(std::vector<uint32_t> & cpts) {
auto result = canonical_decomposition_cpts(cpts, cpts.begin(), cpts.end());
auto result = canonical_decomposition_cpts(cpts, 0);
return sort_by_canonical_class(result);
}

4
unicode.h
View file

@ -17,8 +17,8 @@ std::string unicode_cpt_to_utf8(uint32_t cp);
std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
std::vector<uint32_t> unicode_cpts_normalize_nfd(std::vector<uint32_t> & cpts);
std::vector<uint32_t> canonical_decomposition_cpts(std::vector<uint32_t> & cpts, const std::vector<uint32_t>::iterator& cpt_begin, const std::vector<uint32_t>::iterator& cpt_end);
std::vector<uint32_t> sort_by_canonical_class(const std::vector<uint32_t> & cpts);
std::vector<uint32_t> canonical_decomposition_cpts(std::vector<uint32_t> & cpts, uint32_t starting_offset);
std::vector<uint32_t> sort_by_canonical_class(std::vector<uint32_t> & cpts);
int unicode_cpt_type(uint32_t cp);
int unicode_cpt_type(const std::string & utf8);