vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

move multibyte check to doCompletion

Browse source
This commit is contained in:
anon 2023-06-02 08:16:39 -03:00
parent 8f9e546b51
commit f820740dad
1 changed files with 27 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

56
examples/server/server.cpp
View file

@ -67,6 +67,7 @@ struct llama_server_context
bool verbose = false; bool verbose = false;
int json_indent = -1; int json_indent = -1;
int32_t multibyte_pending = 0;
~llama_server_context() ~llama_server_context()
{ {
@ -82,6 +83,7 @@ struct llama_server_context
generated_text = ""; generated_text = "";
generated_text.reserve(params.n_ctx); generated_text.reserve(params.n_ctx);
stopping_word = ""; stopping_word = "";
multibyte_pending = 0;
n_remain = 0; n_remain = 0;
n_past = 0; n_past = 0;
@ -300,12 +302,32 @@ struct llama_server_context
std::string doCompletion() std::string doCompletion()
{ {
llama_token token = nextToken(); llama_token token = nextToken();
if (token == -1) {
return ""; std::string token_text = token == -1 ? "" : llama_token_to_str(ctx, token);
generated_text += token_text;
if (multibyte_pending > 0) {
multibyte_pending -= token_text.size();
} else if (token_text.size() == 1) {
const char c = token_text[0];
// 2-byte characters: 110xxxxx 10xxxxxx
if ((c & 0xE0) == 0xC0) {
multibyte_pending = 1;
// 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
} else if ((c & 0xF0) == 0xE0) {
multibyte_pending = 2;
// 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
} else if ((c & 0xF8) == 0xF0) {
multibyte_pending = 3;
} else {
multibyte_pending = 0;
}
} }
std::string token_text = llama_token_to_str(ctx, token); if (multibyte_pending > 0 && !has_next_token) {
generated_text += token_text; has_next_token = true;
n_remain++;
}
if (verbose) { if (verbose) {
fprintf(stderr, fprintf(stderr,
@ -794,34 +816,10 @@ int main(int argc, char **argv)
} else { } else {
const auto chunked_content_provider = [&](size_t, DataSink &sink) { const auto chunked_content_provider = [&](size_t, DataSink &sink) {
size_t sent_count = 0; size_t sent_count = 0;
int32_t multibyte_pending = 0;
while (llama.has_next_token) { while (llama.has_next_token) {
const std::string token_text = llama.doCompletion(); const std::string token_text = llama.doCompletion();
if (llama.multibyte_pending > 0) {
if (multibyte_pending > 0) {
multibyte_pending -= token_text.size();
} else if (token_text.size() == 1) {
const char c = token_text[0];
// 2-byte characters: 110xxxxx 10xxxxxx
if ((c & 0xE0) == 0xC0) {
multibyte_pending = 1;
// 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
} else if ((c & 0xF0) == 0xE0) {
multibyte_pending = 2;
// 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
} else if ((c & 0xF8) == 0xF0) {
multibyte_pending = 3;
} else {
multibyte_pending = 0;
}
}
if (multibyte_pending > 0) {
if (!llama.has_next_token) {
llama.has_next_token = true;
llama.n_remain++;
}
continue; continue;
} }