From 581c4a0239ade14e78d515baf4fc5bf400cdd882 Mon Sep 17 00:00:00 2001
From: Georgi Gerganov <ggerganov@gmail.com>
Date: Sat, 27 Apr 2024 18:36:00 +0300
Subject: [PATCH] unicode : try fix windows

---
 unicode.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/unicode.cpp b/unicode.cpp
index d21f3fa46..93e53bc81 100644
--- a/unicode.cpp
+++ b/unicode.cpp
@@ -203,8 +203,13 @@ static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
 }
 
 static inline std::string unicode_wstring_to_utf8(const std::wstring & ws) {
+#if defined(_MSC_VER)
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t> converter;
+    return converter.to_bytes(ws);
+#else
     std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
     return conv.to_bytes(ws);
+#endif
 }
 
 static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) {