From 44dd9ed287c61798b1006e081f34b175e28f8273 Mon Sep 17 00:00:00 2001
From: goerch <jhr.walter@t-online.de>
Date: Mon, 21 Aug 2023 00:28:31 +0200
Subject: [PATCH] Improve commentary

---
 convert.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/convert.py b/convert.py
index cd1798bde..e740560a8 100755
--- a/convert.py
+++ b/convert.py
@@ -737,6 +737,8 @@ class OutputFile:
         tokens = []
         scores = []
         toktypes = []
+        # NOTE: `all_tokens` returns the the base vocabulary and added tokens
+        # TODO: add special tokens?
         for text, score, toktype in vocab.all_tokens():
             tokens.append(text)
             scores.append(score)
@@ -747,8 +749,6 @@ class OutputFile:
         self.gguf.add_token_scores(scores)
         self.gguf.add_token_types(toktypes)
 
-        # TODO: added / special tokens
-
     def add_tensor_info(self, name: str, tensor: LazyTensor) -> None:
         n_elements = 1
         for dim in tensor.shape: