vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add support for I8, I16 and I32 to gguf_writer

Browse source
This commit is contained in:
Ondřej Čertík 2024-03-13 14:06:33 -06:00
parent dc0e4d8e74
commit c5423753f7
1 changed files with 7 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
gguf-py/gguf/gguf_writer.py
View file

@ -208,8 +208,14 @@ class GGUFWriter:
dtype = GGMLQuantizationType.F32 dtype = GGMLQuantizationType.F32
elif tensor_dtype == np.float16: elif tensor_dtype == np.float16:
dtype = GGMLQuantizationType.F16 dtype = GGMLQuantizationType.F16
elif tensor_dtype == np.int8:
dtype = GGMLQuantizationType.I8
elif tensor_dtype == np.int16:
dtype = GGMLQuantizationType.I16
elif tensor_dtype == np.int32:
dtype = GGMLQuantizationType.I32
else: else:
raise ValueError("Only F32 and F16 tensors are supported for now") raise ValueError("Only F32, F16, I8, I16, I32 tensors are supported for now")
else: else:
dtype = raw_dtype dtype = raw_dtype
self.ti_data += self._pack("I", dtype) self.ti_data += self._pack("I", dtype)