vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tokenize antiprompts only once.

Browse source
This commit is contained in:
DAN™ 2024-03-03 08:46:43 -05:00
parent ef651247d0
commit 60325ec78e
1 changed files with 21 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

23
examples/main/main.cpp
View file

@ -760,8 +760,27 @@ int main(int argc, char ** argv) {
? last_output.length() - static_cast<size_t>(antiprompt.length() + extra_padding)
: 0;
auto tmp = ::llama_tokenize(ctx, antiprompt, false, true);
if (last_output.find(antiprompt, search_start_pos) != std::string::npos || (tmp.size() == 1 && llama_sampling_last(ctx_sampling) == tmp[0])) {
if (last_output.find(antiprompt, search_start_pos) != std::string::npos) {
if (params.interactive) {
is_interacting = true;
}
is_antiprompt = true;
break;
}
}
// tokenize reverse/antiprompt special tokens only once using static
static std::vector<std::vector<llama_token>> antiprompt_ids;
if (antiprompt_ids.empty()) {
for (std::string& antiprompt : params.antiprompt) {
antiprompt_ids.push_back(::llama_tokenize(ctx, antiprompt, false, true));
}
}
// check for reverse prompt using special tokens
llama_token last_token = llama_sampling_last(ctx_sampling);
for (std::vector<llama_token> ids : antiprompt_ids) {
if (ids.size() == 1 && last_token == ids[0]) {
if (params.interactive) {
is_interacting = true;
}