vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add files via upload

Browse source
This commit is contained in:
bobqianic 2024-02-13 03:00:05 +00:00 • committed by GitHub
parent 0e5b25b2f2
commit 7570bbcd9b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unicode.h
View file

@ -399,8 +399,7 @@ static int codepoint_type(uint32_t cp) {
static int codepoint_type(const std::string & utf8) {
if (utf8.length() == 0)
return CODEPOINT_TYPE_UNIDENTIFIED;
size_t offset = 0;
return codepoint_type(codepoint_from_utf8(utf8, offset));
return codepoint_type(codepoints_from_utf8(utf8).back());
}
static std::unordered_map<uint8_t, std::string> bytes_to_unicode_map_bpe() {