vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #5 from anon998/stop-stream

Browse source 
Stop generating tokens when the stream is closed.
This commit is contained in:
Randall Fitzgerald 2023-05-30 22:16:32 -04:00 • committed by GitHub
commit 9f2424ac47
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

9
examples/server/server.cpp
View file

@ -668,7 +668,7 @@ bool parse_options_completion(json body, llama_server_context& llama, Response &
std::string tmp_stop = std::string tmp_stop =
std::accumulate(llama.params.antiprompt.begin(), llama.params.antiprompt.end(), std::accumulate(llama.params.antiprompt.begin(), llama.params.antiprompt.end(),
std::string{}, [](std::string a, std::string b) { std::string{}, [](std::string a, std::string b) {
return a + (a != "" ? ", \"" : "") + b + "\""; return a + (a != "" ? ", \"" : "\"") + b + "\"";
}); });
fprintf(stderr, fprintf(stderr,
@ -860,7 +860,12 @@ int main(int argc, char **argv)
data.dump(llama.json_indent, ' ', false, data.dump(llama.json_indent, ' ', false,
json::error_handler_t::replace) + json::error_handler_t::replace) +
"\n\n"; "\n\n";
sink.write(str.data(), str.size()); if (!sink.write(str.data(), str.size())) {
if (llama.verbose) {
fprintf(stderr, "stream closed\n");
}
return false;
}
} }
sink.done(); sink.done();