vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Simplifying an expression.

Browse source
This commit is contained in:
goerch 2023-08-06 13:35:27 +02:00
parent 19e950f051
commit bb6a58d0c3
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
llama.cpp
View file

@ -4096,7 +4096,8 @@ int llama_tokenize_with_model(
llama_token * tokens, llama_token * tokens,
int n_max_tokens, int n_max_tokens,
bool add_bos) { bool add_bos) {
auto res = llama_tokenize(model->vocab, text, add_bos, llama_vocab_type(model->vocab) == "spm"); auto escape = llama_vocab_type(model->vocab) == "spm";
auto res = llama_tokenize(model->vocab, text, add_bos, escape);
if (n_max_tokens < (int) res.size()) { if (n_max_tokens < (int) res.size()) {
return -((int) res.size()); return -((int) res.size());