vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use token type API in test-tokenizer-1.cpp

Browse source
This commit is contained in:
goerch 2023-08-21 19:04:03 +02:00
parent a856685648
commit 11e3806be4
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tests/test-tokenizer-1.cpp
View file

@ -87,8 +87,8 @@ int main(int argc, char **argv) {
return 2; return 2;
} }
} else { } else {
// TODO: needs access to token types llama_token_type type = llama_token_get_type(ctx, i);
if (0 <= i && i < 259) { if (type == LLAMA_TOKEN_TYPE_UNKNOWN || type == LLAMA_TOKEN_TYPE_CONTROL || type == LLAMA_TOKEN_TYPE_BYTE) {
fprintf(stderr, "%s : info: token %d is string %s and bpe returns tokens %s\n", fprintf(stderr, "%s : info: token %d is string %s and bpe returns tokens %s\n",
__func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str()); __func__, i, llama_token_to_str(ctx, i).c_str(), unescape_whitespace(ctx, tokens).c_str());
} else { } else {