vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ggml-quants : faster 1.625 bpw AVX2 vec_dot

Browse source 
Not using a lookup table anymore makes it match q4_0 speed.

* gguf-py : fix formatting

* llama : remove spaces on empty line
This commit is contained in:
Francis Couture-Harpin 2024-06-19 14:34:32 -04:00
parent bd807499f7
commit 7ef4254a92
2 changed files with 58 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

5
gguf-py/gguf/quants.py
View file

@ -148,9 +148,9 @@ def __quantize_q1_3_rows(n: np.ndarray) -> np.ndarray:
q48 = np.sum(q48 * pow3.reshape((1, 1, 4)), axis=2, keepdims=True).reshape((n_blocks, 12))
q4 = np.sum(q4 * pow3.reshape((1, 4)), axis=1, keepdims=True)
q48 = q48 + (q12 * 81)
q = np.concatenate([q48, q4], axis=1);
q = np.concatenate([q48, q4], axis=1)
q = ((q.astype(np.uint16) * 256) // 243).astype(np.uint8)
q = np.where(q != 0, q + 1, 0);
q = np.where(q != 0, q + 1, 0)
return q.reshape(__quantize_q1_3_shape_change(shape))
@ -170,4 +170,3 @@ def quantize_q1_3(data: np.ndarray):
return __quantize_q1_3_lazy(data)
else:
return __quantize_q1_3_array(data)