vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add quantize script for batch quantization

Browse source
This commit is contained in:
Pavol Rusnak 2023-03-13 13:49:46 +01:00
parent 1808ee0500
commit daedf7ce97
No known key found for this signature in database
GPG key ID: 91F3B339B9A02A3D
1 changed files with 15 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

15
quantize.sh Executable file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env bash
if ! [[ "$1" =~ ^[0-9]{1,2}B$ ]]; then
echo
echo "Usage: quantize.sh 7B|13B|30B|65B [--remove-f16]"
echo
exit 1
fi
for i in "models/$1/ggml-model-f16.bin*"; do
./quantize "$i" "${i/f16/q4_0}" 2
if [[ "$2" == "--remove-f16" ]]; then
rm "$i"
fi
done