vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

speculative : add tree-based sampling support

Browse source 
ggml-ci
This commit is contained in:
Georgi Gerganov 2023-10-14 17:54:02 +03:00
parent 5261aee8d8
commit 4de5a2d473
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
11 changed files with 469 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

37
common/log.h
View file

@ -612,6 +612,43 @@ inline std::string log_var_to_string_impl(const std::vector<int> & var)
}() \
.c_str()
#define LOG_BATCH_TOSTR_PRETTY(ctx, batch) \
[&batch, &ctx]() \
{ \
std::stringstream buf; \
buf << "[ "; \
\
bool first = true; \
for (int i = 0; i < batch.n_tokens; ++i) \
{ \
if (!first) \
buf << ", "; \
else \
first = false; \
\
auto detokenized = llama_token_to_piece(ctx, batch.token[i]); \
\
detokenized.erase( \
std::remove_if( \
detokenized.begin(), \
detokenized.end(), \
[](const unsigned char c) { return !std::isprint(c); }), \
detokenized.end()); \
\
buf \
<< "\n" << std::to_string(i) \
<< ":token '" << detokenized << "'" \
<< ":pos " << std::to_string(batch.pos[i]) \
<< ":n_seq_id " << std::to_string(batch.n_seq_id[i]) \
<< ":seq_id " << std::to_string(batch.seq_id[i][0]) \
<< ":logits " << std::to_string(batch.logits[i]); \
} \
buf << " ]"; \
\
return buf.str(); \
}() \
.c_str()
#ifdef LOG_DISABLE_LOGS
#undef LOG