vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

embedding : add warning about missing SEP

Browse source
This commit is contained in:
Georgi Gerganov 2024-05-11 10:44:45 +03:00
parent 23499b81d9
commit 49b3dbbe08
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
examples/embedding/embedding.cpp
View file

@ -129,12 +129,12 @@ int main(int argc, char ** argv) {
inputs.push_back(inp);
}
// add SEP if not present
// JoanFM: I propose to remove this line so that user can make sure that their model is properly configured to tokenize as expected.
// We could also add a parameter, but I think that adding parameters specific for the examples can become messy and unmantaibable easy
// check if the last token is SEP
// it should be automatically added by the tokenizer when 'tokenizer.ggml.add_eos_token' is set to 'true'
for (auto & inp : inputs) {
if (inp.empty() || inp.back() != llama_token_sep(model)) {
inp.push_back(llama_token_sep(model));
fprintf(stderr, "%s: warning: last token in the prompt is not SEP\n", __func__);
fprintf(stderr, "%s: 'tokenizer.ggml.add_eos_token' should be set to 'true' in the GGUF header\n", __func__);
}
}