vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ggml-cpu: support IQ4_NL_4_4 by runtime repack (#10541)

Browse source 
* ggml-cpu: support IQ4_NL_4_4 by runtime repack

* ggml-cpu: add __ARM_FEATURE_DOTPROD guard
This commit is contained in:
Shupei Fan 2024-11-28 20:52:03 +08:00 • committed by GitHub
parent 2025fa67e9
commit c202cef168
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 352 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

6
ggml/src/ggml-common.h
View file

@ -418,6 +418,12 @@ typedef struct {
} block_iq4_xs;
static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");
typedef struct {
ggml_half d[4]; // deltas for 4 iq4_nl blocks
uint8_t qs[QK4_NL * 2];// nibbles / quants for 4 iq4_nl blocks
} block_iq4_nlx4;
static_assert(sizeof(block_iq4_nlx4) == 4 * sizeof(ggml_half) + QK4_NL * 2, "wrong iq4_nlx4 block size/padding");
#endif // GGML_COMMON_DECL
#endif // GGML_COMMON_DECL