vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gguf-py : fix and simplify quantized shape round-trip (#7483)

Browse source 
* gguf-py : fix and simplify quantized shape round-trip

* gguf-py : remove unused import
This commit is contained in:
compilade 2024-05-24 21:11:48 -04:00 • committed by GitHub
parent d041d2ceaa
commit b83bab15a5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 27 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

7
convert-hf-to-gguf.py
View file

@ -313,11 +313,10 @@ class Model:
data = data.astype(np.float32)
data_qtype = gguf.GGMLQuantizationType.F32
block_size, type_size = gguf.GGML_QUANT_SIZES[data_qtype]
shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
# reverse shape to make it similar to the internal ggml dimension order
shape_str = f"""{{{', '.join(str(n) for n in reversed(
(*data.shape[:-1], data.shape[-1] * data.dtype.itemsize // type_size * block_size))
)}}}"""
shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"
# n_dims is implicit in the shape
logger.info(f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}")