vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix debug assertion failure

Browse source
This commit is contained in:
goerch 2023-09-18 20:55:07 +02:00
parent 89e74c67e2
commit 77704232b2
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
unicode.h
View file

@ -250,7 +250,6 @@ static std::string codepoint_to_utf8(uint32_t cp) {
static std::string codepoints_to_utf8(const std::vector<uint32_t>& cps) { static std::string codepoints_to_utf8(const std::vector<uint32_t>& cps) {
std::string result; std::string result;
size_t offset = 0;
for (auto i = 0; i < cps.size(); ++i) { for (auto i = 0; i < cps.size(); ++i) {
result.append(codepoint_to_utf8(cps[i])); result.append(codepoint_to_utf8(cps[i]));
} }
@ -316,7 +315,6 @@ static std::vector<uint16_t> codepoint_to_utf16(uint32_t cp) {
static std::vector<uint16_t> codepoints_to_utf16(const std::vector<uint32_t>& cps) { static std::vector<uint16_t> codepoints_to_utf16(const std::vector<uint32_t>& cps) {
std::vector<uint16_t> result; std::vector<uint16_t> result;
size_t offset = 0;
for (auto i = 0; i < cps.size(); ++i) { for (auto i = 0; i < cps.size(); ++i) {
auto temp = codepoint_to_utf16(cps[i]); auto temp = codepoint_to_utf16(cps[i]);
result.insert(result.end(), temp.begin(), temp.end()); result.insert(result.end(), temp.begin(), temp.end());
@ -349,6 +347,7 @@ static std::vector<uint32_t> codepoints_from_utf16(const std::vector<uint16_t>&
return result; return result;
} }
#define CODEPOINT_TYPE_UNIDENTIFIED 0
#define CODEPOINT_TYPE_DIGIT 1 #define CODEPOINT_TYPE_DIGIT 1
#define CODEPOINT_TYPE_LETTER 2 #define CODEPOINT_TYPE_LETTER 2
#define CODEPOINT_TYPE_WHITESPACE 3 #define CODEPOINT_TYPE_WHITESPACE 3
@ -393,6 +392,8 @@ static int codepoint_type(uint32_t cp) {
} }
static int codepoint_type(std::string utf8) { static int codepoint_type(std::string utf8) {
if (utf8.length() == 0)
return CODEPOINT_TYPE_UNIDENTIFIED;
size_t offset = 0; size_t offset = 0;
return codepoint_type(codepoint_from_utf8(utf8, offset)); return codepoint_type(codepoint_from_utf8(utf8, offset));
} }