vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

preallocate a buffer of fitting size for tokenization (utils.cpp)

Browse source
This commit is contained in:
Green Sky 2023-03-22 12:56:42 +01:00
parent d5850c53ca
commit 7b1b575fe8
No known key found for this signature in database
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
utils.cpp
View file

@ -146,8 +146,10 @@ std::string gpt_random_prompt(std::mt19937 & rng) {
// TODO: not great allocating this every time // TODO: not great allocating this every time
std::vector<llama_token> llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos) { std::vector<llama_token> llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos) {
std::vector<llama_token> res(8096); // initialize to prompt numer of chars, since n_tokens <= n_prompt_chars
std::vector<llama_token> res(text.size() + (int)add_bos);
int n = llama_tokenize(ctx, text.c_str(), res.data(), res.size(), add_bos); int n = llama_tokenize(ctx, text.c_str(), res.data(), res.size(), add_bos);
assert(n >= 0);
res.resize(n); res.resize(n);
return res; return res;