vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added the QK reject message into the quantizer

Browse source
This commit is contained in:
John 2023-06-19 23:05:16 +02:00
parent 1f421dddde
commit 32141a3a75
1 changed files with 15 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

15
libfalcon.cpp
View file

@ -21,6 +21,9 @@
#ifdef GGML_USE_METAL #ifdef GGML_USE_METAL
#include "ggml-metal.h" #include "ggml-metal.h"
#endif #endif
#ifndef QK_K
#define QK_K 256
#endif
#include <array> #include <array>
#include <ctime> #include <ctime>
@ -2558,10 +2561,22 @@ static void falcon_model_quantize_internal(const std::string & fname_inp, const
} else { } else {
new_type = quantized_type; new_type = quantized_type;
#ifdef GGML_USE_K_QUANTS #ifdef GGML_USE_K_QUANTS
if (quantized_type == GGML_TYPE_Q2_K || quantized_type == GGML_TYPE_Q3_K || quantized_type == GGML_TYPE_Q4_K ||
quantized_type == GGML_TYPE_Q5_K || quantized_type == GGML_TYPE_Q6_K) {
int nx = tensor.ne.at(0);
int ny = tensor.ne.at(0);
if (nx % QK_K != 0 || ny % QK_K != 0) {
fprintf(stderr, "\n\n========================= Tensor sizes %d x %d are not divisible by %d\n",nx,ny,QK_K);
fprintf(stderr, "This is required to be able to use k-quants for now!\n");
fprintf(stderr, "========================================================================================\n\n");
throw std::runtime_error("Unsupported tensor size encountered\n");
}
}
// if (tensor.name == ".mlp.dense_") { // if (tensor.name == ".mlp.dense_") {
// new_type = GGML_TYPE_Q6_K; // new_type = GGML_TYPE_Q6_K;
// } // }
// TODO falcon // TODO falcon
#endif #endif
float * f32_data; float * f32_data;