Improve commentary
This commit is contained in:
parent
6586487e62
commit
44dd9ed287
1 changed files with 2 additions and 2 deletions
|
@ -737,6 +737,8 @@ class OutputFile:
|
|||
tokens = []
|
||||
scores = []
|
||||
toktypes = []
|
||||
# NOTE: `all_tokens` returns the the base vocabulary and added tokens
|
||||
# TODO: add special tokens?
|
||||
for text, score, toktype in vocab.all_tokens():
|
||||
tokens.append(text)
|
||||
scores.append(score)
|
||||
|
@ -747,8 +749,6 @@ class OutputFile:
|
|||
self.gguf.add_token_scores(scores)
|
||||
self.gguf.add_token_types(toktypes)
|
||||
|
||||
# TODO: added / special tokens
|
||||
|
||||
def add_tensor_info(self, name: str, tensor: LazyTensor) -> None:
|
||||
n_elements = 1
|
||||
for dim in tensor.shape:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue