vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support of converting local models added to convert-hf-to-gguf-update.py

Browse source
This commit is contained in:
Paul Rock 2024-05-26 22:14:32 +03:00
parent dff451cfa1
commit 9f6bedea7b
1 changed files with 21 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

27
convert-hf-to-gguf-update.py
View file

@ -26,6 +26,7 @@ import logging
import os import os
import pathlib import pathlib
import re import re
import shutil
import requests import requests
import sys import sys
@ -106,12 +107,26 @@ def download_model(model):
if tokt == TOKENIZER_TYPE.SPM: if tokt == TOKENIZER_TYPE.SPM:
files.append("tokenizer.model") files.append("tokenizer.model")
for file in files: if os.path.isdir(repo):
save_path = f"models/tokenizers/{name}/{file}" # Если repo это путь на файловой системе, копируем директорию
if os.path.isfile(save_path): for file in files:
logger.info(f"{name}: File {save_path} already exists - skipping") src_path = os.path.join(repo, file)
continue dst_path = f"models/tokenizers/{name}/{file}"
download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path) if os.path.isfile(dst_path):
logger.info(f"{name}: File {dst_path} already exists - skipping")
continue
if os.path.isfile(src_path):
shutil.copy2(src_path, dst_path)
logger.info(f"{name}: Copied {src_path} to {dst_path}")
else:
logger.warning(f"{name}: Source file {src_path} does not exist")
else:
for file in files:
save_path = f"models/tokenizers/{name}/{file}"
if os.path.isfile(save_path):
logger.info(f"{name}: File {save_path} already exists - skipping")
continue
download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path)
for model in models: for model in models: