vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix masking in __compute_fp32_to_bf16

Browse source
This commit is contained in:
Sigbjørn Skjæret 2024-06-14 11:06:21 +02:00 • committed by GitHub
parent 46054d1aab
commit 069369f3fe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
gguf-py/gguf/quants.py
View file

@ -25,14 +25,14 @@ def quant_shape_from_byte_shape(shape: Sequence[int], quant_type: GGMLQuantizati
# same as ggml_compute_fp32_to_bf16 in ggml-impl.h # same as ggml_compute_fp32_to_bf16 in ggml-impl.h
def __compute_fp32_to_bf16(n: np.ndarray) -> np.ndarray: def __compute_fp32_to_bf16(n: np.ndarray) -> np.ndarray:
n = n.astype(np.float32, copy=False).view(np.int32) n = n.astype(np.float32, copy=False).view(np.uint32)
# force nan to quiet # force nan to quiet
n = np.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n) n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | (64 << 16), n)
# flush subnormals to zero # flush subnormals to zero
n = np.where((n & 0x7f800000) == 0, n & 0x80000000, n) n = np.where((n & 0x7f800000) == 0, n & np.uint32(0x80000000), n)
# round to nearest even # round to nearest even
n = (n + (0x7fff + ((n >> 16) & 1))) >> 16 n = (n + (0x7fff + ((n >> 16) & 1))) >> 16
return n.astype(np.int16) return n.astype(np.uint16)
# for fp32 values that are just extended bf16 # for fp32 values that are just extended bf16
@ -55,10 +55,10 @@ def __apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.
def __quantize_bf16_array(n: np.ndarray) -> np.ndarray: def __quantize_bf16_array(n: np.ndarray) -> np.ndarray:
return __apply_over_grouped_rows(__compute_fp32_to_bf16, arr=n, otype=np.int16, oshape=n.shape) return __apply_over_grouped_rows(__compute_fp32_to_bf16, arr=n, otype=np.uint16, oshape=n.shape)
__quantize_bf16_lazy = LazyNumpyTensor._wrap_fn(__quantize_bf16_array, meta_noop=np.int16) __quantize_bf16_lazy = LazyNumpyTensor._wrap_fn(__quantize_bf16_array, meta_noop=np.uint16)
def quantize_bf16(n: np.ndarray): def quantize_bf16(n: np.ndarray):