vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gguf : write tokenizer data

Browse source
This commit is contained in:
M. Yusuf Sarıgöz 2023-07-29 13:30:22 +03:00
parent 8a76dd8a85
commit cc3dd7f042
1 changed files with 9 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

9
gguf.py
View file

@ -267,6 +267,15 @@ class GGUFWriter:
def write_rope_scale(self, llm: str, value: float): def write_rope_scale(self, llm: str, value: float):
self.write_float32(constants.KEY_ROPE_SCALE.format(llm=llm), value) self.write_float32(constants.KEY_ROPE_SCALE.format(llm=llm), value)
def write_tokenizer_model(self, model: str):
self.write_string(constants.KEY_TOKENIZER_MODEL, model)
def write_token_list(self, tokens: List[str]):
self.write_array(constants.KEY_TOKENIZER_LIST, tokens)
def write_token_scores(self, scores: List[float]):
self.write_array(constants.KEY_TOKENIZER_SCORES, scores)
# Example usage: # Example usage:
if __name__ == "__main__": if __name__ == "__main__":