From 7570bbcd9be160dafff43cb94af94ff6c68be816 Mon Sep 17 00:00:00 2001
From: bobqianic <129547291+bobqianic@users.noreply.github.com>
Date: Tue, 13 Feb 2024 03:00:05 +0000
Subject: [PATCH] Add files via upload

---
 unicode.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/unicode.h b/unicode.h
index 844eff3da..5940291bc 100644
--- a/unicode.h
+++ b/unicode.h
@@ -399,8 +399,7 @@ static int codepoint_type(uint32_t cp) {
 static int codepoint_type(const std::string & utf8) {
     if (utf8.length() == 0)
         return CODEPOINT_TYPE_UNIDENTIFIED;
-    size_t offset = 0;
-    return codepoint_type(codepoint_from_utf8(utf8, offset));
+    return codepoint_type(codepoints_from_utf8(utf8).back());
 }
 
 static std::unordered_map<uint8_t, std::string> bytes_to_unicode_map_bpe() {