vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update unicode.h

Browse source 
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
goerch 2023-10-02 09:19:06 +02:00 • committed by GitHub
parent a9a2af93ed
commit dccd1db48e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unicode.h
View file

@ -293,8 +293,9 @@ static uint32_t codepoint_from_utf8(const std::string& utf8, size_t& offset) {
static std::vector<uint32_t> codepoints_from_utf8(const std::string& utf8) { static std::vector<uint32_t> codepoints_from_utf8(const std::string& utf8) {
std::vector<uint32_t> result; std::vector<uint32_t> result;
size_t offset = 0; size_t offset = 0;
while (offset < utf8.size()) while (offset < utf8.size()) {
result.push_back(codepoint_from_utf8(utf8, offset)); result.push_back(codepoint_from_utf8(utf8, offset));
}
return result; return result;
} }