vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: Add method for generating the checksums and writing the results to a json file

Browse source
This commit is contained in:
teleprint-me 2024-05-18 01:46:13 -04:00
parent 302258721b
commit b2ca23c746
No known key found for this signature in database
GPG key ID: B0D11345E65C4D48
1 changed files with 12 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

13
gguf-py/gguf/huggingface_hub.py
View file

@ -176,4 +176,15 @@ class HFTokenizerRequest:
self.resolve_tokenizer_model(filename, filepath, model) self.resolve_tokenizer_model(filename, filepath, model)
def generate_checksums(self) -> None: def generate_checksums(self) -> None:
pass checksums = []
for model in self.models:
mapping = {}
filepath = f"{self.local_path}/{model['repo']}"
tokenizer = AutoTokenizer.from_pretrained(filepath, trust_remote=True)
mapping.update(model)
mapping['checksum'] = sha256(str(tokenizer.vocab).encode()).hexdigest()
self.logger.info(f"Hashed {model['repo']} as {mapping['checksum']}")
checksums.append(mapping)
with open(f"{self.local_path.parent}/checksums.json") as file:
json.dump(checksums, file)