From 1dc13168ff313681b0946b6be85042f43aef685d Mon Sep 17 00:00:00 2001
From: goerch <jhr.walter@t-online.de>
Date: Sun, 22 Oct 2023 18:59:06 +0200
Subject: [PATCH] Remove unnecessary restriction in test case

---
 tests/test-tokenizer-1-bpe.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test-tokenizer-1-bpe.cpp b/tests/test-tokenizer-1-bpe.cpp
index fd0d1556a..85a59a14d 100644
--- a/tests/test-tokenizer-1-bpe.cpp
+++ b/tests/test-tokenizer-1-bpe.cpp
@@ -62,9 +62,6 @@ int main(int argc, char **argv) {
     const int n_vocab = llama_n_vocab(model);
 
     for (int i = 0; i < n_vocab; ++i) {
-        if (llama_token_get_type(ctx, i) == LLAMA_TOKEN_TYPE_USER_DEFINED) {
-            continue;
-        }
         std::string str = llama_detokenize_bpe(ctx, std::vector<int>(1, i));
         try {
             auto cps = codepoints_from_utf8(str);