vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #6 from WangHaoranRobin/robin_fork_master

Browse source 
server: remove n_probs upper limit of 5
This commit is contained in:
WangHaoranRobin 2023-06-25 14:16:35 -07:00 • committed by GitHub
commit bd6550bd8b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
examples/server/server.cpp
View file

@ -101,7 +101,7 @@ static void server_log(const char * level, const char * function, int line,
static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) { static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) {
std::string out = token == -1 ? "" : llama_token_to_str(ctx, token); std::string out = token == -1 ? "" : llama_token_to_str(ctx, token);
// if first bit is 1, meaning it's a partial character // if first bit is 1, meaning it's a partial character
if ((out[0] & 0x80) == 0x80) { if (out.size() > 0 && (out[0] & 0x80) == 0x80) {
std::stringstream ss; std::stringstream ss;
ss<< std::hex << (out[0] & 0xff); ss<< std::hex << (out[0] & 0xff);
std::string res ( ss.str() ); std::string res ( ss.str() );
@ -382,8 +382,8 @@ struct llama_server_context {
result.tok = llama_sample_token(ctx, &candidates_p); result.tok = llama_sample_token(ctx, &candidates_p);
} }
} }
// Add maximum of 5 most probable tokens to the result
for (size_t i = 0; i < std::min(candidates_p.size, std::min((size_t) n_probs, size_t(5))); ++i) { for (size_t i = 0; i < std::min(candidates_p.size, (size_t) n_probs); ++i) {
result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p}); result.probs.push_back({candidates_p.data[i].id, candidates_p.data[i].p});
} }
last_n_tokens.erase(last_n_tokens.begin()); last_n_tokens.erase(last_n_tokens.begin());