vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Truncate prompt if longer than context + n_predict

Browse source
This commit is contained in:
Matthew McAllister 2023-03-12 21:49:17 -07:00
parent b9bd1d0141
commit 1133eea479
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
main.cpp
View file

@ -782,8 +782,10 @@ int main(int argc, char ** argv) {
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::llama_tokenize(vocab, params.prompt, true);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
if (embd_inp.size() + params.n_predict > model.hparams.n_ctx) {
int offset = embd_inp.size() - model.hparams.n_ctx + params.n_predict;
embd_inp = std::vector<gpt_vocab::id>(embd_inp.begin() + offset, embd_inp.end());
}
printf("\n");
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());