From 7e0b6a7b3ba94ff624dc27c1e0e735fded8819b8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 8 May 2024 12:47:07 +0300 Subject: [PATCH] py : also print the normalizers --- convert-hf-to-gguf-update.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index ae901e24c..a3fe67ee7 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -151,6 +151,8 @@ for model in models: # print the "pre_tokenizer" content from the tokenizer.json with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f: cfg = json.load(f) + normalizer = cfg["normalizer"] + logger.info("normalizer: " + json.dumps(normalizer, indent=4)) pre_tokenizer = cfg["pre_tokenizer"] logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))