vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

handle wide characters in llama_file examples

Browse source
This commit is contained in:
Bruce MacDonald 2024-03-11 11:13:35 -04:00
parent 8a11598b1e
commit 18504b6a34
4 changed files with 84 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

21
common/train.cpp
View file

@ -716,7 +716,28 @@ struct llama_file {
size_t size; size_t size;
llama_file(const char * fname, const char * mode) { llama_file(const char * fname, const char * mode) {
#ifdef _WIN32
// temporarily change the locale to the system default to handle Unicode file names
std::string oldLocale = std::setlocale(LC_ALL, nullptr);
std::setlocale(LC_ALL, "");
// convert multi-byte string to wide-char string
int wsize = MultiByteToWideChar(CP_UTF8, 0, fname, -1, nullptr, 0);
std::vector<wchar_t> wfname(wsize);
MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wsize);
// determine the correct wide-character mode string
std::wstring wmode;
for(; *mode; ++mode) {
wmode += wchar_t(*mode);
}
fp = _wfopen(wfname.data(), wmode.c_str());
std::setlocale(LC_ALL, oldLocale.c_str());
#else
fp = std::fopen(fname, mode); fp = std::fopen(fname, mode);
#endif
if (fp == NULL) { if (fp == NULL) {
size = 0; size = 0;
} else { } else {

21
examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp
View file

@ -464,7 +464,28 @@ struct llama_file {
size_t size; size_t size;
llama_file(const char * fname, const char * mode) { llama_file(const char * fname, const char * mode) {
#ifdef _WIN32
// temporarily change the locale to the system default to handle Unicode file names
std::string oldLocale = std::setlocale(LC_ALL, nullptr);
std::setlocale(LC_ALL, "");
// convert multi-byte string to wide-char string
int wsize = MultiByteToWideChar(CP_UTF8, 0, fname, -1, nullptr, 0);
std::vector<wchar_t> wfname(wsize);
MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wsize);
// determine the correct wide-character mode string
std::wstring wmode;
for(; *mode; ++mode) {
wmode += wchar_t(*mode);
}
fp = _wfopen(wfname.data(), wmode.c_str());
std::setlocale(LC_ALL, oldLocale.c_str());
#else
fp = std::fopen(fname, mode); fp = std::fopen(fname, mode);
#endif
if (fp == NULL) { if (fp == NULL) {
size = 0; size = 0;
} else { } else {

21
examples/finetune/finetune.cpp
View file

@ -952,7 +952,28 @@ struct llama_file {
size_t size; size_t size;
llama_file(const char * fname, const char * mode) { llama_file(const char * fname, const char * mode) {
#ifdef _WIN32
// temporarily change the locale to the system default to handle Unicode file names
std::string oldLocale = std::setlocale(LC_ALL, nullptr);
std::setlocale(LC_ALL, "");
// convert multi-byte string to wide-char string
int wsize = MultiByteToWideChar(CP_UTF8, 0, fname, -1, nullptr, 0);
std::vector<wchar_t> wfname(wsize);
MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wsize);
// determine the correct wide-character mode string
std::wstring wmode;
for(; *mode; ++mode) {
wmode += wchar_t(*mode);
}
fp = _wfopen(wfname.data(), wmode.c_str());
std::setlocale(LC_ALL, oldLocale.c_str());
#else
fp = std::fopen(fname, mode); fp = std::fopen(fname, mode);
#endif
if (fp == NULL) { if (fp == NULL) {
size = 0; size = 0;
} else { } else {

21
llama.cpp
View file

@ -988,7 +988,28 @@ struct llama_file {
size_t size; size_t size;
llama_file(const char * fname, const char * mode) { llama_file(const char * fname, const char * mode) {
#ifdef _WIN32
// temporarily change the locale to the system default to handle Unicode file names
std::string oldLocale = std::setlocale(LC_ALL, nullptr);
std::setlocale(LC_ALL, "");
// convert multi-byte string to wide-char string
int wsize = MultiByteToWideChar(CP_UTF8, 0, fname, -1, nullptr, 0);
std::vector<wchar_t> wfname(wsize);
MultiByteToWideChar(CP_UTF8, 0, fname, -1, wfname.data(), wsize);
// determine the correct wide-character mode string
std::wstring wmode;
for(; *mode; ++mode) {
wmode += wchar_t(*mode);
}
fp = _wfopen(wfname.data(), wmode.c_str());
std::setlocale(LC_ALL, oldLocale.c_str());
#else
fp = std::fopen(fname, mode); fp = std::fopen(fname, mode);
#endif
if (fp == NULL) { if (fp == NULL) {
throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno))); throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
} }