vbatts/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updated file conversion and simplified activation logic

Browse source
This commit is contained in:
chooper1 2023-09-12 22:44:51 -07:00
parent 11f72245ff
commit 7954f8defd
6 changed files with 1445 additions and 1330 deletions
Download patch file Download diff file Expand all files Collapse all files

17
convert-llama-ggml-to-gguf.py
View file

@ -33,7 +33,6 @@ GGML_QUANT_SIZES = {
gguf.GGMLQuantizationType.Q5_K : (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),
gguf.GGMLQuantizationType.Q6_K : (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),
gguf.GGMLQuantizationType.Q8_K : (256, 4 + QK_K + QK_K // 8),
gguf.GGMLQuantizationType.Q4_SQ : (1, 4),
}
class GGMLFormat(IntEnum):
@ -59,7 +58,6 @@ class GGMLFType(IntEnum):
MOSTLY_Q5_K_S = 16
MOSTLY_Q5_K_M = 17
MOSTLY_Q6_K = 18
MOSTLY_Q4_SQ = 19
class Hyperparameters:
def __init__(self):
@ -122,7 +120,7 @@ class Tensor:
self.len_bytes = np.int64(0)
self.use_padding = use_padding
def load(self, data, offset, squeezellm=False):
def load(self, data, offset):
orig_offset = offset
(n_dims, name_len, dtype) = struct.unpack('<3I', data[offset:offset + 12])
assert n_dims >= 0 and n_dims <= 4, f'Invalid tensor dimensions {n_dims}'
@ -139,9 +137,6 @@ class Tensor:
pad = ((offset + 31) & ~31) - offset if self.use_padding else 0
offset += pad
n_elems = np.prod(self.dims)
if squeezellm and n_dims > 1 and dtype == gguf.GGMLQuantizationType.Q4_SQ:
n_elems = n_elems / 8
n_elems += self.dims[1] * 8 # add 16 fp16 elements per row
n_bytes = np.int64(np.int64(n_elems) * np.int64(tysize)) // np.int64(blksize)
self.start_offset = offset
self.len_bytes = n_bytes
@ -191,20 +186,19 @@ class GGMLModel:
if len(err) > 0:
raise ValueError(f'{err} Sorry, your {self.file_format.name}v{self.format_version} file of type {ftype.name} is not eligible for conversion.')
def load(self, data, offset, squeezellm=False):
def load(self, data, offset):
offset += self.validate_header(data, offset)
hp = Hyperparameters()
offset += hp.load(data, offset)
print(f'* File format: {self.file_format.name}v{self.format_version} with ftype {hp.ftype.name}')
if not squeezellm:
self.validate_conversion(hp.ftype)
self.validate_conversion(hp.ftype)
vocab = Vocab(load_scores = self.file_format > GGMLFormat.GGML)
offset += vocab.load(data, offset, hp.n_vocab)
tensors: list[Tensor] = []
tensor_map = {}
while offset < len(data):
tensor = Tensor(use_padding = self.file_format > GGMLFormat.GGMF)
offset += tensor.load(data, offset, squeezellm=squeezellm)
offset += tensor.load(data, offset)
tensor_map[tensor.name] = len(tensors)
tensors.append(tensor)
self.hyperparameters = hp
@ -420,7 +414,6 @@ def handle_args():
help="directory containing tokenizer.model, if separate from model file - only meaningful with --model-metadata-dir")
parser.add_argument("--vocabtype", choices=["spm", "bpe"], default="spm",
help="vocab format - only meaningful with --model-metadata-dir and/or --vocab-dir (default: spm)")
parser.add_argument("--squeezellm", action="store_true", help="Convert to SQLLM")
return parser.parse_args()
def main():
@ -432,7 +425,7 @@ def main():
data = np.memmap(cfg.input, mode = 'r')
model = GGMLModel()
print('* Scanning GGML input file')
offset = model.load(data, 0, cfg.squeezellm)
offset = model.load(data, 0)
print(f'* GGML model hyperparameters: {model.hyperparameters}')
vocab_override = None
params_override = None

1297
convert-sqllm-to-ggml.py
View file

File diff suppressed because it is too large Load diff

1433
convert-sqllm-to-gguf.py Normal file
View file

File diff suppressed because it is too large Load diff

22
ggml.c
View file

@ -1790,10 +1790,10 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
.type_size = sizeof(int32_t),
.is_quantized = true,
.to_float = NULL,
.from_float = NULL,
.from_float = (ggml_from_float_t) ggml_fp32_to_fp16_row,
.from_float_reference = NULL,
.vec_dot = ggml_vec_dot_q4_sq_fp16,
.vec_dot_type = GGML_TYPE_F32,
.vec_dot_type = GGML_TYPE_F16,
}
#endif
};
@ -11351,8 +11351,9 @@ static void ggml_compute_forward_mul_mat(
}
#endif
if (params->type == GGML_TASK_INIT && src0->type != GGML_TYPE_Q4_SQ) {
if (params->type == GGML_TASK_INIT){
if (src1->type != vec_dot_type) {
char * wdata = params->wdata;
const size_t row_size = ne10*ggml_type_size(vec_dot_type)/ggml_blck_size(vec_dot_type);
@ -11366,21 +11367,6 @@ static void ggml_compute_forward_mul_mat(
}
}
return;
} else if (params->type == GGML_TASK_INIT) { //SQLLM - copy fp32 vec over
ggml_fp16_t * wdata = params->wdata;
float * srcvec;
for (int64_t i13 = 0; i13 < ne13; ++i13) {
for (int64_t i12 = 0; i12 < ne12; ++i12) {
for (int64_t i11 = 0; i11 < ne11; ++i11) {
srcvec = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
for (int64_t i10 = 0; i10 < ne10; ++i10) {
*wdata = ggml_fp32_to_fp16(srcvec[i10]);
wdata += 1;
}
}
}
}
return;
}

4
ggml.h
View file

@ -304,7 +304,7 @@ extern "C" {
GGML_TYPE_Q5_K = 13,
GGML_TYPE_Q6_K = 14,
GGML_TYPE_Q8_K = 15,
GGML_TYPE_Q4_SQ = 16,
GGML_TYPE_Q4_SQ = 19,
GGML_TYPE_I8,
GGML_TYPE_I16,
GGML_TYPE_I32,
@ -333,7 +333,7 @@ extern "C" {
GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
GGML_FTYPE_MOSTLY_Q4_SQ = 16, // except 1d tensors
GGML_FTYPE_MOSTLY_Q4_SQ = 19, // except 1d tensors
};
// available tensor operations:

2
gguf-py/gguf/gguf.py
View file

@ -389,7 +389,7 @@ class GGMLQuantizationType(IntEnum):
Q5_K = 13
Q6_K = 14
Q8_K = 15
Q4_SQ = 16
Q4_SQ = 19
class GGUFValueType(IntEnum):