vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove unnecessary restriction in test case

Browse source
This commit is contained in:
goerch 2023-10-22 18:59:06 +02:00
parent 8ee736370a
commit 1dc13168ff
1 changed files with 0 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

3
tests/test-tokenizer-1-bpe.cpp
View file

@ -62,9 +62,6 @@ int main(int argc, char **argv) {
const int n_vocab = llama_n_vocab(model);
for (int i = 0; i < n_vocab; ++i) {
if (llama_token_get_type(ctx, i) == LLAMA_TOKEN_TYPE_USER_DEFINED) {
continue;
}
std::string str = llama_detokenize_bpe(ctx, std::vector<int>(1, i));
try {
auto cps = codepoints_from_utf8(str);