vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve commentary

Browse source
This commit is contained in:
goerch 2023-08-21 00:28:31 +02:00
parent 6586487e62
commit 44dd9ed287
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
convert.py
View file

@ -737,6 +737,8 @@ class OutputFile:
tokens = [] tokens = []
scores = [] scores = []
toktypes = [] toktypes = []
# NOTE: `all_tokens` returns the the base vocabulary and added tokens
# TODO: add special tokens?
for text, score, toktype in vocab.all_tokens(): for text, score, toktype in vocab.all_tokens():
tokens.append(text) tokens.append(text)
scores.append(score) scores.append(score)
@ -747,8 +749,6 @@ class OutputFile:
self.gguf.add_token_scores(scores) self.gguf.add_token_scores(scores)
self.gguf.add_token_types(toktypes) self.gguf.add_token_types(toktypes)
# TODO: added / special tokens
def add_tensor_info(self, name: str, tensor: LazyTensor) -> None: def add_tensor_info(self, name: str, tensor: LazyTensor) -> None:
n_elements = 1 n_elements = 1
for dim in tensor.shape: for dim in tensor.shape: