From 4f46e29b09d53722c7d73e447dd84fd02cb91abd Mon Sep 17 00:00:00 2001
From: Georgi Gerganov <ggerganov@gmail.com>
Date: Sun, 13 Oct 2024 13:42:16 +0300
Subject: [PATCH] llama : print more info about control tokens

---
 src/llama.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/llama.cpp b/src/llama.cpp
index 80cc93931..1813dd29b 100644
--- a/src/llama.cpp
+++ b/src/llama.cpp
@@ -6725,10 +6725,6 @@ static void llm_load_vocab(
             vocab.special_eog_ids.insert(vocab.special_eom_id);
             LLAMA_LOG_WARN("%s: special_eom_id is not in special_eog_ids - the tokenizer config may be incorrect\n", __func__);
         }
-
-        if (vocab.special_fim_sep_id != LLAMA_TOKEN_NULL && vocab.special_eog_ids.count(vocab.special_fim_sep_id) == 0) {
-            vocab.special_eog_ids.insert(vocab.special_fim_sep_id);
-        }
     }
 
     // build special tokens cache