vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use _wfopen instead of fopen on Windows (#6248)

Browse source 
also fix missing #defines before windows.h, and BPE LF token on MSVC
This commit is contained in:
Jared Van Bortel 2024-03-23 18:48:02 -04:00 • committed by GitHub
parent 95562175f8
commit 94d1b3b411
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 69 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama.cpp
View file

@ -1065,7 +1065,7 @@ struct llama_file {
size_t size;
llama_file(const char * fname, const char * mode) {
fp = std::fopen(fname, mode);
fp = ggml_fopen(fname, mode);
if (fp == NULL) {
throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
}
@ -4006,7 +4006,7 @@ static void llm_load_vocab(
} else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) {
vocab.linefeed_id = vocab.special_pad_id;
} else {
const std::vector<int> ids = llama_tokenize_internal(vocab, "\u010A", false);
const std::vector<int> ids = llama_tokenize_internal(vocab, "\xC4\x8A", false); // U+010A
GGML_ASSERT(!ids.empty() && "model vocab missing newline token");
vocab.linefeed_id = ids[0];
}