vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Style

Browse source
This commit is contained in:
jaime-m-p 2024-04-30 13:18:25 +02:00
parent 2cd1eb0daa
commit 0c6d820b89
1 changed files with 33 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

55
unicode.cpp
View file

@ -228,7 +228,6 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
size_t start = 0; size_t start = 0;
for (auto offset : offsets) { for (auto offset : offsets) {
const size_t offset_ini = start; const size_t offset_ini = start;
const size_t offset_end = start + offset; const size_t offset_end = start + offset;
assert(offset_end <= cpts.size()); assert(offset_end <= cpts.size());
@ -246,10 +245,11 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
auto _add_token = [&] (const size_t end) -> size_t { auto _add_token = [&] (const size_t end) -> size_t {
assert(_prev_end <= end && end <= offset_end); assert(_prev_end <= end && end <= offset_end);
size_t len = end - _prev_end; size_t len = end - _prev_end;
if(len > 0) if (len > 0) {
bpe_offsets.push_back(len); bpe_offsets.push_back(len);
}
_prev_end = end; _prev_end = end;
//if(len) { //if (len > 0) {
// std::string s = ""; // std::string s = "";
// for(size_t p = end-len; p < end; p++) // for(size_t p = end-len; p < end; p++)
// s += unicode_cpt_to_utf8(cpts[p]); // s += unicode_cpt_to_utf8(cpts[p]);
@ -258,7 +258,7 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
return len; return len;
}; };
for(size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
const char32_t cpt = _get_cpt(pos); const char32_t cpt = _get_cpt(pos);
const int cpt_type = _get_cpt_type(pos); const int cpt_type = _get_cpt_type(pos);
@ -268,7 +268,8 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
pos += _add_token(pos+2); pos += _add_token(pos+2);
continue; continue;
} else if (pos+2 < offset_end) { }
if (pos+2 < offset_end) {
char32_t cpt_next_next = _get_cpt(pos+2); char32_t cpt_next_next = _get_cpt(pos+2);
if ((cpt_next == 'r' && cpt_next_next == 'e') || if ((cpt_next == 'r' && cpt_next_next == 'e') ||
(cpt_next == 'v' && cpt_next_next == 'e') || (cpt_next == 'v' && cpt_next_next == 'e') ||
@ -283,24 +284,27 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
// regex: <space>?\p{L}+ // regex: <space>?\p{L}+
if (cpt2_type == CODEPOINT_TYPE_LETTER) { if (cpt2_type == CODEPOINT_TYPE_LETTER) {
pos += (cpt == ' '); pos += (cpt == ' ');
while(cpt2_type == CODEPOINT_TYPE_LETTER) while (cpt2_type == CODEPOINT_TYPE_LETTER) {
cpt2_type = _get_cpt_type(++pos); cpt2_type = _get_cpt_type(++pos);
}
_add_token(pos); _add_token(pos);
continue; continue;
} }
// regex: <space>?\p{N}+ // regex: <space>?\p{N}+
if (cpt2_type == CODEPOINT_TYPE_DIGIT) { if (cpt2_type == CODEPOINT_TYPE_DIGIT) {
pos += (cpt == ' '); pos += (cpt == ' ');
while(cpt2_type == CODEPOINT_TYPE_DIGIT) while (cpt2_type == CODEPOINT_TYPE_DIGIT) {
cpt2_type = _get_cpt_type(++pos); cpt2_type = _get_cpt_type(++pos);
}
_add_token(pos); _add_token(pos);
continue; continue;
} }
// regex: <space>?[^\s\p{L}\p{N}]+ // regex: <space>?[^\s\p{L}\p{N}]+
if (cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) { if (cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) {
pos += (cpt == ' '); pos += (cpt == ' ');
while(cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) while (cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) {
cpt2_type = _get_cpt_type(++pos); cpt2_type = _get_cpt_type(++pos);
}
_add_token(pos); _add_token(pos);
continue; continue;
} }
@ -311,14 +315,14 @@ static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & t
} }
// regex: \s+(?!\S) // regex: \s+(?!\S)
if(num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != 0) { if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != 0) {
pos += num_whitespaces - 1; pos += num_whitespaces - 1;
_add_token(pos); _add_token(pos);
continue; continue;
} }
// regex: \s+ // regex: \s+
if(num_whitespaces > 0) { if (num_whitespaces > 0) {
pos += num_whitespaces; pos += num_whitespaces;
_add_token(pos); _add_token(pos);
continue; continue;
@ -341,7 +345,6 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
size_t start = 0; size_t start = 0;
for (auto offset : offsets) { for (auto offset : offsets) {
const size_t offset_ini = start; const size_t offset_ini = start;
const size_t offset_end = start + offset; const size_t offset_end = start + offset;
assert(offset_end <= cpts.size()); assert(offset_end <= cpts.size());
@ -359,10 +362,11 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
auto _add_token = [&] (const size_t end) -> size_t { auto _add_token = [&] (const size_t end) -> size_t {
assert(_prev_end <= end && end <= offset_end); assert(_prev_end <= end && end <= offset_end);
size_t len = end - _prev_end; size_t len = end - _prev_end;
if(len > 0) if (len > 0) {
bpe_offsets.push_back(len); bpe_offsets.push_back(len);
}
_prev_end = end; _prev_end = end;
//if(len) { //if (len > 0) {
// std::string s = ""; // std::string s = "";
// for(size_t p = end-len; p < end; p++) // for(size_t p = end-len; p < end; p++)
// s += unicode_cpt_to_utf8(cpts[p]); // s += unicode_cpt_to_utf8(cpts[p]);
@ -371,7 +375,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
return len; return len;
}; };
for(size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) {
const char32_t cpt = _get_cpt(pos); const char32_t cpt = _get_cpt(pos);
const int cpt_type = _get_cpt_type(pos); const int cpt_type = _get_cpt_type(pos);
@ -381,7 +385,8 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') {
pos += _add_token(pos+2); pos += _add_token(pos+2);
continue; continue;
} else if (pos+2 < offset_end) { }
if (pos+2 < offset_end) {
char32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); char32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2));
if ((cpt_next == 'r' && cpt_next_next == 'e') || if ((cpt_next == 'r' && cpt_next_next == 'e') ||
(cpt_next == 'v' && cpt_next_next == 'e') || (cpt_next == 'v' && cpt_next_next == 'e') ||
@ -394,10 +399,11 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
// regex: [^\r\n\p{L}\p{N}]?\p{L}+ //####FIXME: the first \p{L} is correct? // regex: [^\r\n\p{L}\p{N}]?\p{L}+ //####FIXME: the first \p{L} is correct?
if (cpt != '\r' && cpt != '\n' && /*cpt_type != CODEPOINT_TYPE_LETTER &&*/ cpt_type != CODEPOINT_TYPE_DIGIT) { if (cpt != '\r' && cpt != '\n' && /*cpt_type != CODEPOINT_TYPE_LETTER &&*/ cpt_type != CODEPOINT_TYPE_DIGIT) {
if(cpt_type == CODEPOINT_TYPE_LETTER || _get_cpt_type(pos+1) == CODEPOINT_TYPE_LETTER) { // one or more letters if (cpt_type == CODEPOINT_TYPE_LETTER || _get_cpt_type(pos+1) == CODEPOINT_TYPE_LETTER) { // one or more letters
pos++; pos++;
while(_get_cpt_type(pos) == CODEPOINT_TYPE_LETTER) while (_get_cpt_type(pos) == CODEPOINT_TYPE_LETTER) {
pos++; pos++;
}
_add_token(pos); _add_token(pos);
continue; continue;
} }
@ -406,7 +412,7 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
// regex: \p{N}{1,3} // regex: \p{N}{1,3}
if (cpt_type == CODEPOINT_TYPE_DIGIT) { if (cpt_type == CODEPOINT_TYPE_DIGIT) {
size_t ini = pos; size_t ini = pos;
while(_get_cpt_type(pos) == CODEPOINT_TYPE_DIGIT) { while (_get_cpt_type(pos) == CODEPOINT_TYPE_DIGIT) {
if (++pos - ini >= 3 ) { if (++pos - ini >= 3 ) {
_add_token(pos); _add_token(pos);
ini = pos; ini = pos;
@ -420,11 +426,13 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
int cpt2_type = (cpt == ' ' ? _get_cpt_type(pos+1) : cpt_type); int cpt2_type = (cpt == ' ' ? _get_cpt_type(pos+1) : cpt_type);
if (cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) { if (cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) {
pos += (cpt == ' '); pos += (cpt == ' ');
while(cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) while (cpt2_type != CODEPOINT_TYPE_WHITESPACE && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_DIGIT && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) {
cpt2_type = _get_cpt_type(++pos); cpt2_type = _get_cpt_type(++pos);
}
char32_t cpt2 = _get_cpt(pos); char32_t cpt2 = _get_cpt(pos);
while(cpt2 == '\r' || cpt2 == '\n') while (cpt2 == '\r' || cpt2 == '\n') {
cpt2 = _get_cpt(++pos); cpt2 = _get_cpt(++pos);
}
_add_token(pos); _add_token(pos);
continue; continue;
} }
@ -433,8 +441,9 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
size_t last_end_r_or_n = 0; size_t last_end_r_or_n = 0;
while (_get_cpt_type(pos+num_whitespaces) == CODEPOINT_TYPE_WHITESPACE) { while (_get_cpt_type(pos+num_whitespaces) == CODEPOINT_TYPE_WHITESPACE) {
char32_t cpt2 = _get_cpt(pos+num_whitespaces); char32_t cpt2 = _get_cpt(pos+num_whitespaces);
if (cpt2 == '\r' || cpt2 == '\n') if (cpt2 == '\r' || cpt2 == '\n') {
last_end_r_or_n = pos + num_whitespaces + 1; last_end_r_or_n = pos + num_whitespaces + 1;
}
num_whitespaces++; num_whitespaces++;
} }
@ -446,14 +455,14 @@ static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string &
} }
// regex: \s+(?!\S) // regex: \s+(?!\S)
if(num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != 0) { if (num_whitespaces > 1 && _get_cpt(pos+num_whitespaces) != 0) {
pos += num_whitespaces - 1; pos += num_whitespaces - 1;
_add_token(pos); _add_token(pos);
continue; continue;
} }
// regex: \s+ // regex: \s+
if(num_whitespaces > 0) { if (num_whitespaces > 0) {
pos += num_whitespaces; pos += num_whitespaces;
_add_token(pos); _add_token(pos);
continue; continue;