vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add quantize-stats command for testing quantization (#728)

Browse source 
Command that calculates some statistics over the errors introduced by
quantization, like mean square error, max error and some percentile errors for layer
weights. Should be useful for testing quantization improvements.

Exposes some internal state from ggml and llama for testing
This commit is contained in:
unbounded 2023-04-08 00:09:18 +02:00 • committed by GitHub
parent 698f7b5d63
commit 62cfc54f77
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 415 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

30
ggml.c
View file

@ -6564,29 +6564,27 @@ static void ggml_compute_forward_mul_mat_f16_f32(
//}
}
typedef void (*dequantize_row_q_t)(const void * restrict x, float * restrict y, int k);
typedef void (*quantize_row_q_t)(const float * restrict x, void * restrict y, int k);
typedef void (*vec_dot_q_t)(const int n, float * restrict s, const void * restrict x, const void * restrict y);
typedef struct {
dequantize_row_q_t dequantize_row_q;
quantize_row_q_t quantize_row_q;
vec_dot_q_t vec_dot_q;
} quantize_fns_t;
static const quantize_fns_t quantize_fns[GGML_TYPE_COUNT] = {
[GGML_TYPE_Q4_0] = {
.dequantize_row_q = dequantize_row_q4_0,
.quantize_row_q = quantize_row_q4_0,
.vec_dot_q = ggml_vec_dot_q4_0,
.dequantize_row_q = dequantize_row_q4_0,
.quantize_row_q = quantize_row_q4_0,
.quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_0_reference,
.vec_dot_q = ggml_vec_dot_q4_0,
},
[GGML_TYPE_Q4_1] = {
.dequantize_row_q = dequantize_row_q4_1,
.quantize_row_q = quantize_row_q4_1,
.vec_dot_q = ggml_vec_dot_q4_1,
.dequantize_row_q = dequantize_row_q4_1,
.quantize_row_q = quantize_row_q4_1,
.quantize_row_q_reference = (quantize_row_q_t) quantize_row_q4_1_reference,
.vec_dot_q = ggml_vec_dot_q4_1,
},
};
// For internal test use
quantize_fns_t ggml_internal_get_quantize_fn(size_t i) {
GGML_ASSERT(i < GGML_TYPE_COUNT);
return quantize_fns[i];
}
static void ggml_compute_forward_mul_mat_q_f32(
const struct ggml_compute_params * params,
const struct ggml_tensor * src0,