vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BERT WIP

Browse source
This commit is contained in:
Jared Van Bortel 2024-02-06 17:03:12 -05:00
parent f57fadc009
commit 7286b83d3f
5 changed files with 310 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

3
gguf-py/gguf/gguf_writer.py
View file

@ -387,6 +387,9 @@ class GGUFWriter:
def add_token_types(self, types: Sequence[TokenType] | Sequence[int]) -> None:
self.add_array(Keys.Tokenizer.TOKEN_TYPE, types)
def add_token_type_count(self, value: int) -> None:
self.add_uint32(Keys.Tokenizer.TOKEN_TYPE_COUNT, value)
def add_token_scores(self, scores: Sequence[float]) -> None:
self.add_array(Keys.Tokenizer.SCORES, scores)