convert : add "tokenizer.ggml.pre" GGUF KV (wip)

2024-04-26 19:21:55 +03:00 · 2024-04-26 19:21:55 +03:00 · 9b4d63ae53
commit 9b4d63ae53
parent e3f6dc7409
6 changed files with 58 additions and 7 deletions
--- a/convert-llama-ggml-to-gguf.py
+++ b/convert-llama-ggml-to-gguf.py
@ -281,6 +281,7 @@ class GGMLToGGUF:
    def add_vocab(self, gguf_writer):
        hp = self.model.hyperparameters
        gguf_writer.add_tokenizer_model('llama')
+        gguf_writer.add_tokenizer_pre('default')
        tokens = []
        scores = []
        toktypes = []