vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support handling of LFS for download.

Browse source
This commit is contained in:
DAN™ 2024-05-02 07:17:05 -04:00
parent a34ace9f52
commit 8242447b7b
1 changed files with 8 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
convert-hf-to-gguf-update.py
View file

@ -95,6 +95,14 @@ for model in models:
save_path = f"models/tokenizers/{name}/tokenizer.json"
download_file_with_auth(url, token, save_path)
# if downloaded file is less than 1KB, we likely need to download an LFS instead
if os.path.getsize(save_path) < 1024:
# remove the file
os.remove(save_path)
url = f"{repo}/resolve/main/tokenizer.json"
save_path = f"models/tokenizers/{name}/tokenizer.json"
download_file_with_auth(url, token, save_path)
if tokt == TOKENIZER_TYPE.SPM:
url = f"{repo}/resolve/main/tokenizer.model"
save_path = f"models/tokenizers/{name}/tokenizer.model"