vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

revert the wstring tokenization. coherency was affected

Browse source
This commit is contained in:
Concedo 2023-06-24 12:58:49 +08:00
parent 6da38b0d40
commit 8342fe81b1
2 changed files with 9 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

2
koboldcpp.py
View file

@ -225,7 +225,7 @@ maxhordectx = 1024
maxhordelen = 256
modelbusy = False
defaultport = 5001
KcppVersion = "1.32.2"
KcppVersion = "1.32.3"
showdebug = True
class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):

16
otherarch/mpt_v3.cpp
View file

@ -87,14 +87,14 @@ bool mpt_model_load(const std::string & fname, mpt_model & model, gpt_vocab & vo
word.assign(buf.data(), len);
// Convert token from utf-8
std::wstring word_multibytes = convert_to_wstring(word);
if(word_multibytes!=L"")
{
word.resize(word_multibytes.size());
for (int w = 0; w < word_multibytes.size(); w++) {
word[w] = uint8_t(word_multibytes[w]);
}
}
// std::wstring word_multibytes = convert_to_wstring(word);
// if(word_multibytes!=L"")
// {
// word.resize(word_multibytes.size());
// for (int w = 0; w < word_multibytes.size(); w++) {
// word[w] = uint8_t(word_multibytes[w]);
// }
// }
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;