vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

convert : use utf8 encoding

Browse source
This commit is contained in:
Georgi Gerganov 2024-04-30 10:20:18 +03:00
parent 8843a98c2b
commit 5a91d63d8b
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
convert-hf-to-gguf-update.py
View file

@ -128,7 +128,7 @@ for model in models:
print(f"chkhsh: {chkhsh}") print(f"chkhsh: {chkhsh}")
# print the "pre_tokenizer" content from the tokenizer.json # print the "pre_tokenizer" content from the tokenizer.json
with open(f"models/tokenizers/{name}/tokenizer.json", "r") as f: with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:
cfg = json.load(f) cfg = json.load(f)
pre_tokenizer = cfg["pre_tokenizer"] pre_tokenizer = cfg["pre_tokenizer"]
print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4)) print("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))
@ -249,7 +249,7 @@ for model in models:
from transformers import AutoTokenizer from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}") tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")
with open(f"models/ggml-vocab-{name}.gguf.inp", "w") as f: with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:
for text in tests: for text in tests:
f.write(f"{text}") f.write(f"{text}")
f.write("\n__ggml_vocab_test__\n") f.write("\n__ggml_vocab_test__\n")