vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add files via upload

Browse source
This commit is contained in:
Black-Engineer 2023-03-16 08:23:02 +02:00 • committed by GitHub
parent 9b4a15b17d
commit fcafaf03b0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
llama.exe Normal file
View file

Binary file not shown.

BIN
quantize.exe Normal file
View file

Binary file not shown.

15
quantize.py Normal file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env python3
import os
import sys
if not (len(sys.argv) == 2 and sys.argv[1] in ["7B", "13B", "30B", "65B"]):
print(f"\nUsage: {sys.argv[0]} 7B|13B|30B|65B [--remove-f16]\n")
sys.exit(1)
for i in os.listdir(f"models/{sys.argv[1]}"):
if i.endswith("ggml-model-f16.bin.1"):
os.system(f"quantize.exe {os.path.join('models', sys.argv[1], i)} {os.path.join('models', sys.argv[1], i.replace('f16', 'q4_0'))} 2")
print(f"quantize.exe {os.path.join('models', sys.argv[1], i)} {os.path.join('models', sys.argv[1], i.replace('f16', 'q4_1'))} 2")
if len(sys.argv) == 3 and sys.argv[2] == "--remove-f16":
os.remove(os.path.join('models', sys.argv[1], i))