vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

embedding : parse special tokens

Browse source
This commit is contained in:
Georgi Gerganov 2024-09-25 20:36:38 +03:00
parent 7bde9a0452
commit c62a39d91e
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/embedding/embedding.cpp
View file

@ -135,7 +135,7 @@ int main(int argc, char ** argv) {
// tokenize the prompts and trim // tokenize the prompts and trim
std::vector<std::vector<int32_t>> inputs; std::vector<std::vector<int32_t>> inputs;
for (const auto & prompt : prompts) { for (const auto & prompt : prompts) {
auto inp = ::llama_tokenize(ctx, prompt, true, false); auto inp = ::llama_tokenize(ctx, prompt, true, true);
if (inp.size() > n_batch) { if (inp.size() > n_batch) {
LOG_ERR("%s: number of tokens in input line (%lld) exceeds batch size (%lld), increase batch size and re-run\n", LOG_ERR("%s: number of tokens in input line (%lld) exceeds batch size (%lld), increase batch size and re-run\n",
__func__, (long long int) inp.size(), (long long int) n_batch); __func__, (long long int) inp.size(), (long long int) n_batch);